2024-11-17 12:45:20,110 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-17 12:45:20,120 main DEBUG Took 0.008592 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 12:45:20,120 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 12:45:20,121 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 12:45:20,121 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 12:45:20,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,129 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 12:45:20,139 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,141 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,141 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,142 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,143 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,143 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,144 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,145 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,145 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,146 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,147 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,147 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,148 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 12:45:20,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,150 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 12:45:20,151 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 12:45:20,152 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 12:45:20,154 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 12:45:20,154 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 12:45:20,155 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 12:45:20,156 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 12:45:20,163 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 12:45:20,166 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 12:45:20,167 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 12:45:20,167 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 12:45:20,168 main DEBUG createAppenders(={Console}) 2024-11-17 12:45:20,168 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-17 12:45:20,169 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-17 12:45:20,169 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-17 12:45:20,169 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 12:45:20,170 main DEBUG OutputStream closed 2024-11-17 12:45:20,170 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 12:45:20,170 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 12:45:20,170 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-17 12:45:20,240 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 12:45:20,242 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 12:45:20,243 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 12:45:20,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 12:45:20,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 12:45:20,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 12:45:20,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 12:45:20,245 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 12:45:20,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 12:45:20,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 12:45:20,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 12:45:20,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 12:45:20,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 12:45:20,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 12:45:20,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 12:45:20,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 12:45:20,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 12:45:20,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 12:45:20,251 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 12:45:20,251 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-17 12:45:20,252 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 12:45:20,252 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-17T12:45:20,486 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df 2024-11-17 12:45:20,489 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 12:45:20,490 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T12:45:20,497 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-17T12:45:20,505 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-17T12:45:20,528 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T12:45:20,567 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T12:45:20,568 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T12:45:20,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T12:45:20,595 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e, deleteOnExit=true 2024-11-17T12:45:20,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T12:45:20,596 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/test.cache.data in system properties and HBase conf 2024-11-17T12:45:20,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T12:45:20,597 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir in system properties and HBase conf 2024-11-17T12:45:20,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T12:45:20,598 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T12:45:20,599 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T12:45:20,686 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T12:45:20,773 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T12:45:20,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T12:45:20,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T12:45:20,777 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T12:45:20,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T12:45:20,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T12:45:20,779 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T12:45:20,779 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T12:45:20,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T12:45:20,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T12:45:20,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/nfs.dump.dir in system properties and HBase conf 2024-11-17T12:45:20,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/java.io.tmpdir in system properties and HBase conf 2024-11-17T12:45:20,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T12:45:20,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T12:45:20,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T12:45:21,807 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T12:45:21,879 INFO [Time-limited test {}] log.Log(170): Logging initialized @2464ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T12:45:21,946 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T12:45:22,012 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T12:45:22,043 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T12:45:22,043 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T12:45:22,044 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T12:45:22,057 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T12:45:22,060 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,AVAILABLE} 2024-11-17T12:45:22,060 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T12:45:22,216 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5608f426{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/java.io.tmpdir/jetty-localhost-44985-hadoop-hdfs-3_4_1-tests_jar-_-any-1823892101883699826/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T12:45:22,222 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:44985} 2024-11-17T12:45:22,222 INFO [Time-limited test {}] server.Server(415): Started @2807ms 2024-11-17T12:45:22,712 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T12:45:22,719 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T12:45:22,720 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T12:45:22,721 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T12:45:22,721 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T12:45:22,721 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cf706e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,AVAILABLE} 2024-11-17T12:45:22,722 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e9da5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T12:45:22,822 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b57af8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/java.io.tmpdir/jetty-localhost-41939-hadoop-hdfs-3_4_1-tests_jar-_-any-2890980542204632180/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T12:45:22,823 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@461b5ea8{HTTP/1.1, (http/1.1)}{localhost:41939} 2024-11-17T12:45:22,823 INFO [Time-limited test {}] server.Server(415): Started @3408ms 2024-11-17T12:45:22,869 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T12:45:22,979 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T12:45:22,986 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T12:45:22,990 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T12:45:22,990 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T12:45:22,990 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T12:45:22,992 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b034666{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,AVAILABLE} 2024-11-17T12:45:22,993 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40a3a086{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T12:45:23,095 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44df5d9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/java.io.tmpdir/jetty-localhost-34113-hadoop-hdfs-3_4_1-tests_jar-_-any-5192881977378048172/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T12:45:23,096 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@56fbb88e{HTTP/1.1, (http/1.1)}{localhost:34113} 2024-11-17T12:45:23,096 INFO [Time-limited test {}] server.Server(415): Started @3681ms 2024-11-17T12:45:23,099 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T12:45:23,139 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T12:45:23,145 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T12:45:23,146 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T12:45:23,147 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T12:45:23,147 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T12:45:23,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7161498b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,AVAILABLE} 2024-11-17T12:45:23,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1846121a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T12:45:23,248 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f0cddd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/java.io.tmpdir/jetty-localhost-35837-hadoop-hdfs-3_4_1-tests_jar-_-any-4165375882098184894/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T12:45:23,249 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@25cdce3{HTTP/1.1, (http/1.1)}{localhost:35837} 2024-11-17T12:45:23,249 INFO [Time-limited test {}] server.Server(415): Started @3834ms 2024-11-17T12:45:23,251 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T12:45:24,203 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data1/current/BP-131233367-172.17.0.2-1731847521320/current, will proceed with Du for space computation calculation, 2024-11-17T12:45:24,203 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data3/current/BP-131233367-172.17.0.2-1731847521320/current, will proceed with Du for space computation calculation, 2024-11-17T12:45:24,203 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data4/current/BP-131233367-172.17.0.2-1731847521320/current, will proceed with Du for space computation calculation, 2024-11-17T12:45:24,203 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data2/current/BP-131233367-172.17.0.2-1731847521320/current, will proceed with Du for space computation calculation, 2024-11-17T12:45:24,232 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T12:45:24,232 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T12:45:24,251 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data5/current/BP-131233367-172.17.0.2-1731847521320/current, will proceed with Du for space computation calculation, 2024-11-17T12:45:24,251 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data6/current/BP-131233367-172.17.0.2-1731847521320/current, will proceed with Du for space computation calculation, 2024-11-17T12:45:24,270 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T12:45:24,290 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa96f750c9f77a7fc with lease ID 0x1ff9d812c6eca1b0: Processing first storage report for DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df from datanode DatanodeRegistration(127.0.0.1:42393, datanodeUuid=6fb5cbbe-7ba9-44a1-9898-0f10a7eae599, infoPort=33427, infoSecurePort=0, ipcPort=46451, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320) 2024-11-17T12:45:24,291 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa96f750c9f77a7fc with lease ID 0x1ff9d812c6eca1b0: from storage DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df node DatanodeRegistration(127.0.0.1:42393, datanodeUuid=6fb5cbbe-7ba9-44a1-9898-0f10a7eae599, infoPort=33427, infoSecurePort=0, ipcPort=46451, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-17T12:45:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6746df286c6aa7ab with lease ID 0x1ff9d812c6eca1ae: Processing first storage report for DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c from datanode DatanodeRegistration(127.0.0.1:34421, datanodeUuid=9612c832-297d-4827-8928-65664340be6b, infoPort=38819, infoSecurePort=0, ipcPort=34587, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320) 2024-11-17T12:45:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6746df286c6aa7ab with lease ID 0x1ff9d812c6eca1ae: from storage DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c node DatanodeRegistration(127.0.0.1:34421, datanodeUuid=9612c832-297d-4827-8928-65664340be6b, infoPort=38819, infoSecurePort=0, ipcPort=34587, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T12:45:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa206af1b97aa5f68 with lease ID 0x1ff9d812c6eca1af: Processing first storage report for DS-82575f11-d40e-49b1-b3c2-7b9c594a815d from datanode DatanodeRegistration(127.0.0.1:44483, datanodeUuid=3e61f543-d328-43aa-92e4-5687ae6794a4, infoPort=42965, infoSecurePort=0, ipcPort=39299, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320) 2024-11-17T12:45:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa206af1b97aa5f68 with lease ID 0x1ff9d812c6eca1af: from storage DS-82575f11-d40e-49b1-b3c2-7b9c594a815d node DatanodeRegistration(127.0.0.1:44483, datanodeUuid=3e61f543-d328-43aa-92e4-5687ae6794a4, infoPort=42965, infoSecurePort=0, ipcPort=39299, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T12:45:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa96f750c9f77a7fc with lease ID 0x1ff9d812c6eca1b0: Processing first storage report for DS-a376a9de-045c-48a7-ac1a-0e99f6ba8974 from datanode DatanodeRegistration(127.0.0.1:42393, datanodeUuid=6fb5cbbe-7ba9-44a1-9898-0f10a7eae599, infoPort=33427, infoSecurePort=0, ipcPort=46451, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320) 2024-11-17T12:45:24,292 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa96f750c9f77a7fc with lease ID 0x1ff9d812c6eca1b0: from storage DS-a376a9de-045c-48a7-ac1a-0e99f6ba8974 node DatanodeRegistration(127.0.0.1:42393, datanodeUuid=6fb5cbbe-7ba9-44a1-9898-0f10a7eae599, infoPort=33427, infoSecurePort=0, ipcPort=46451, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T12:45:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6746df286c6aa7ab with lease ID 0x1ff9d812c6eca1ae: Processing first storage report for DS-d2dee138-9529-4da4-95d1-3ae9e0dcae03 from datanode DatanodeRegistration(127.0.0.1:34421, datanodeUuid=9612c832-297d-4827-8928-65664340be6b, infoPort=38819, infoSecurePort=0, ipcPort=34587, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320) 2024-11-17T12:45:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6746df286c6aa7ab with lease ID 0x1ff9d812c6eca1ae: from storage DS-d2dee138-9529-4da4-95d1-3ae9e0dcae03 node DatanodeRegistration(127.0.0.1:34421, datanodeUuid=9612c832-297d-4827-8928-65664340be6b, infoPort=38819, infoSecurePort=0, ipcPort=34587, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T12:45:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa206af1b97aa5f68 with lease ID 0x1ff9d812c6eca1af: Processing first storage report for DS-9664f132-1290-485a-b8dd-2a0a4f0d69b8 from datanode DatanodeRegistration(127.0.0.1:44483, datanodeUuid=3e61f543-d328-43aa-92e4-5687ae6794a4, infoPort=42965, infoSecurePort=0, ipcPort=39299, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320) 2024-11-17T12:45:24,293 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa206af1b97aa5f68 with lease ID 0x1ff9d812c6eca1af: from storage DS-9664f132-1290-485a-b8dd-2a0a4f0d69b8 node DatanodeRegistration(127.0.0.1:44483, datanodeUuid=3e61f543-d328-43aa-92e4-5687ae6794a4, infoPort=42965, infoSecurePort=0, ipcPort=39299, storageInfo=lv=-57;cid=testClusterID;nsid=839136856;c=1731847521320), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T12:45:24,381 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df 2024-11-17T12:45:24,458 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/zookeeper_0, clientPort=59814, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T12:45:24,465 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59814 2024-11-17T12:45:24,475 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:24,477 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:24,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741825_1001 (size=7) 2024-11-17T12:45:24,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741825_1001 (size=7) 2024-11-17T12:45:24,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741825_1001 (size=7) 2024-11-17T12:45:25,072 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8 with version=8 2024-11-17T12:45:25,072 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/hbase-staging 2024-11-17T12:45:25,374 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0d751a622496:0 server-side Connection retries=45 2024-11-17T12:45:25,383 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:25,384 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:25,388 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T12:45:25,388 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:25,389 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T12:45:25,517 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T12:45:25,571 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T12:45:25,580 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T12:45:25,583 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T12:45:25,607 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21648 (auto-detected) 2024-11-17T12:45:25,608 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T12:45:25,626 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38549 2024-11-17T12:45:25,645 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38549 connecting to ZooKeeper ensemble=127.0.0.1:59814 2024-11-17T12:45:25,750 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:385490x0, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T12:45:25,754 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38549-0x1014a1da31f0000 connected 2024-11-17T12:45:25,852 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:25,857 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:25,868 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T12:45:25,872 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8, hbase.cluster.distributed=false 2024-11-17T12:45:25,893 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T12:45:25,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38549 2024-11-17T12:45:25,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38549 2024-11-17T12:45:25,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38549 2024-11-17T12:45:25,902 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38549 2024-11-17T12:45:25,903 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38549 2024-11-17T12:45:25,995 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45 2024-11-17T12:45:25,996 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:25,996 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:25,996 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T12:45:25,997 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:25,997 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T12:45:25,999 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T12:45:26,001 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T12:45:26,002 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38707 2024-11-17T12:45:26,004 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38707 connecting to ZooKeeper ensemble=127.0.0.1:59814 2024-11-17T12:45:26,005 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:26,008 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:26,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387070x0, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T12:45:26,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:387070x0, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T12:45:26,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38707-0x1014a1da31f0001 connected 2024-11-17T12:45:26,047 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T12:45:26,056 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T12:45:26,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T12:45:26,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T12:45:26,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38707 2024-11-17T12:45:26,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38707 2024-11-17T12:45:26,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38707 2024-11-17T12:45:26,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38707 2024-11-17T12:45:26,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38707 2024-11-17T12:45:26,081 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45 2024-11-17T12:45:26,081 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:26,081 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:26,082 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T12:45:26,082 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:26,082 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T12:45:26,082 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T12:45:26,082 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T12:45:26,083 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44359 2024-11-17T12:45:26,085 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44359 connecting to ZooKeeper ensemble=127.0.0.1:59814 2024-11-17T12:45:26,086 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:26,088 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:26,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:443590x0, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T12:45:26,109 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:443590x0, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T12:45:26,109 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44359-0x1014a1da31f0002 connected 2024-11-17T12:45:26,110 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T12:45:26,111 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T12:45:26,112 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T12:45:26,113 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T12:45:26,114 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44359 2024-11-17T12:45:26,114 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44359 2024-11-17T12:45:26,114 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44359 2024-11-17T12:45:26,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44359 2024-11-17T12:45:26,115 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44359 2024-11-17T12:45:26,132 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0d751a622496:0 server-side Connection retries=45 2024-11-17T12:45:26,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:26,132 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:26,132 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T12:45:26,133 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T12:45:26,133 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T12:45:26,133 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T12:45:26,133 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T12:45:26,134 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32849 2024-11-17T12:45:26,136 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32849 connecting to ZooKeeper ensemble=127.0.0.1:59814 2024-11-17T12:45:26,137 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:26,139 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:26,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:328490x0, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T12:45:26,150 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:328490x0, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T12:45:26,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32849-0x1014a1da31f0003 connected 2024-11-17T12:45:26,151 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T12:45:26,152 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T12:45:26,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T12:45:26,154 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T12:45:26,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32849 2024-11-17T12:45:26,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32849 2024-11-17T12:45:26,158 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32849 2024-11-17T12:45:26,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32849 2024-11-17T12:45:26,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32849 2024-11-17T12:45:26,173 DEBUG [M:0;0d751a622496:38549 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0d751a622496:38549 2024-11-17T12:45:26,174 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0d751a622496,38549,1731847525223 2024-11-17T12:45:26,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,186 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0d751a622496,38549,1731847525223 2024-11-17T12:45:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T12:45:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T12:45:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T12:45:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,216 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T12:45:26,217 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0d751a622496,38549,1731847525223 from backup master directory 2024-11-17T12:45:26,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0d751a622496,38549,1731847525223 2024-11-17T12:45:26,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T12:45:26,226 WARN [master/0d751a622496:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T12:45:26,226 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0d751a622496,38549,1731847525223 2024-11-17T12:45:26,229 INFO [master/0d751a622496:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T12:45:26,230 INFO [master/0d751a622496:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T12:45:26,279 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/hbase.id] with ID: e64d9651-704a-4260-becd-b88a4609dc90 2024-11-17T12:45:26,279 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/.tmp/hbase.id 2024-11-17T12:45:26,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741826_1002 (size=42) 2024-11-17T12:45:26,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741826_1002 (size=42) 2024-11-17T12:45:26,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741826_1002 (size=42) 2024-11-17T12:45:26,294 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/.tmp/hbase.id]:[hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/hbase.id] 2024-11-17T12:45:26,340 INFO [master/0d751a622496:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T12:45:26,344 INFO [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T12:45:26,362 INFO [master/0d751a622496:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-17T12:45:26,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,383 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:26,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741827_1003 (size=196) 2024-11-17T12:45:26,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741827_1003 (size=196) 2024-11-17T12:45:26,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741827_1003 (size=196) 2024-11-17T12:45:26,421 INFO [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T12:45:26,423 INFO [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T12:45:26,428 INFO [master/0d751a622496:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:26,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741828_1004 (size=1189) 2024-11-17T12:45:26,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741828_1004 (size=1189) 2024-11-17T12:45:26,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741828_1004 (size=1189) 2024-11-17T12:45:26,475 INFO [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store 2024-11-17T12:45:26,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741829_1005 (size=34) 2024-11-17T12:45:26,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741829_1005 (size=34) 2024-11-17T12:45:26,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741829_1005 (size=34) 2024-11-17T12:45:26,502 INFO [master/0d751a622496:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T12:45:26,505 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:26,506 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T12:45:26,506 INFO [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:45:26,506 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:45:26,507 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T12:45:26,508 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:45:26,508 INFO [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:45:26,509 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731847526505Disabling compacts and flushes for region at 1731847526506 (+1 ms)Disabling writes for close at 1731847526507 (+1 ms)Writing region close event to WAL at 1731847526508 (+1 ms)Closed at 1731847526508 2024-11-17T12:45:26,511 WARN [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/.initializing 2024-11-17T12:45:26,511 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/WALs/0d751a622496,38549,1731847525223 2024-11-17T12:45:26,519 INFO [master/0d751a622496:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:26,534 INFO [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C38549%2C1731847525223, suffix=, logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/WALs/0d751a622496,38549,1731847525223, archiveDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/oldWALs, maxLogs=10 2024-11-17T12:45:26,561 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/WALs/0d751a622496,38549,1731847525223/0d751a622496%2C38549%2C1731847525223.1731847526539, exclude list is [], retry=0 2024-11-17T12:45:26,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:26,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:26,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:26,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:26,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T12:45:26,621 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-17T12:45:26,687 INFO [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/WALs/0d751a622496,38549,1731847525223/0d751a622496%2C38549%2C1731847525223.1731847526539 2024-11-17T12:45:26,688 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:26,688 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:26,689 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:26,691 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,692 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T12:45:26,749 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:26,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:26,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,756 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T12:45:26,757 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:26,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:26,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T12:45:26,761 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:26,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:26,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,764 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T12:45:26,764 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:26,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:26,766 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,769 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,770 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,777 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,778 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,781 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T12:45:26,785 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T12:45:26,790 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:26,791 INFO [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61649312, jitterRate=-0.08135366439819336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T12:45:26,799 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731847526702Initializing all the Stores at 1731847526705 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847526705Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847526706 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847526706Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847526706Cleaning up temporary data from old regions at 1731847526778 (+72 ms)Region opened successfully at 1731847526799 (+21 ms) 2024-11-17T12:45:26,800 INFO [master/0d751a622496:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T12:45:26,832 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c912195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0 2024-11-17T12:45:26,861 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T12:45:26,871 INFO [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T12:45:26,871 INFO [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T12:45:26,873 INFO [master/0d751a622496:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T12:45:26,875 INFO [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T12:45:26,879 INFO [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T12:45:26,879 INFO [master/0d751a622496:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T12:45:26,902 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T12:45:26,910 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T12:45:26,950 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T12:45:26,953 INFO [master/0d751a622496:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T12:45:26,955 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T12:45:26,966 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T12:45:26,968 INFO [master/0d751a622496:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T12:45:26,973 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T12:45:26,982 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T12:45:26,984 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T12:45:26,991 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T12:45:27,015 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T12:45:27,024 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,037 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0d751a622496,38549,1731847525223, sessionid=0x1014a1da31f0000, setting cluster-up flag (Was=false) 2024-11-17T12:45:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,091 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T12:45:27,095 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,38549,1731847525223 2024-11-17T12:45:27,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,141 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T12:45:27,143 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0d751a622496,38549,1731847525223 2024-11-17T12:45:27,152 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T12:45:27,163 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(746): ClusterId : e64d9651-704a-4260-becd-b88a4609dc90 2024-11-17T12:45:27,163 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(746): ClusterId : e64d9651-704a-4260-becd-b88a4609dc90 2024-11-17T12:45:27,163 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(746): ClusterId : e64d9651-704a-4260-becd-b88a4609dc90 2024-11-17T12:45:27,165 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T12:45:27,165 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T12:45:27,165 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T12:45:27,176 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T12:45:27,176 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T12:45:27,176 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T12:45:27,176 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T12:45:27,176 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T12:45:27,176 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T12:45:27,184 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T12:45:27,184 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T12:45:27,184 DEBUG [RS:2;0d751a622496:32849 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7960c3fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0 2024-11-17T12:45:27,184 DEBUG [RS:1;0d751a622496:44359 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4984409e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0 2024-11-17T12:45:27,186 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T12:45:27,186 DEBUG [RS:0;0d751a622496:38707 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554ffcc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0d751a622496/172.17.0.2:0 2024-11-17T12:45:27,201 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0d751a622496:32849 2024-11-17T12:45:27,203 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0d751a622496:38707 2024-11-17T12:45:27,203 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0d751a622496:44359 2024-11-17T12:45:27,205 INFO [RS:0;0d751a622496:38707 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T12:45:27,205 INFO [RS:2;0d751a622496:32849 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T12:45:27,205 INFO [RS:1;0d751a622496:44359 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T12:45:27,205 INFO [RS:0;0d751a622496:38707 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T12:45:27,205 INFO [RS:1;0d751a622496:44359 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T12:45:27,205 INFO [RS:2;0d751a622496:32849 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T12:45:27,205 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T12:45:27,205 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T12:45:27,205 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T12:45:27,208 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,38549,1731847525223 with port=32849, startcode=1731847526132 2024-11-17T12:45:27,208 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,38549,1731847525223 with port=44359, startcode=1731847526081 2024-11-17T12:45:27,208 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,38549,1731847525223 with port=38707, startcode=1731847525964 2024-11-17T12:45:27,221 DEBUG [RS:2;0d751a622496:32849 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T12:45:27,221 DEBUG [RS:0;0d751a622496:38707 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T12:45:27,221 DEBUG [RS:1;0d751a622496:44359 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T12:45:27,234 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T12:45:27,246 INFO [master/0d751a622496:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T12:45:27,254 INFO [master/0d751a622496:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T12:45:27,257 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50169, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T12:45:27,257 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53221, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T12:45:27,257 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40831, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T12:45:27,261 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0d751a622496,38549,1731847525223 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T12:45:27,264 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T12:45:27,269 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5 2024-11-17T12:45:27,269 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T12:45:27,269 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5 2024-11-17T12:45:27,270 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5 2024-11-17T12:45:27,270 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0d751a622496:0, corePoolSize=5, maxPoolSize=5 2024-11-17T12:45:27,270 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T12:45:27,270 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0d751a622496:0, corePoolSize=10, maxPoolSize=10 2024-11-17T12:45:27,270 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,270 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0d751a622496:0, corePoolSize=2, maxPoolSize=2 2024-11-17T12:45:27,270 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,279 INFO [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731847557278 2024-11-17T12:45:27,281 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T12:45:27,281 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T12:45:27,281 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T12:45:27,282 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T12:45:27,285 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T12:45:27,285 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T12:45:27,286 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T12:45:27,286 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T12:45:27,286 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:27,286 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T12:45:27,286 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,290 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T12:45:27,292 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T12:45:27,292 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T12:45:27,295 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T12:45:27,295 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T12:45:27,295 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T12:45:27,295 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T12:45:27,295 WARN [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T12:45:27,295 WARN [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T12:45:27,295 WARN [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T12:45:27,295 INFO [master/0d751a622496:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T12:45:27,315 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847527315,5,FailOnTimeoutGroup] 2024-11-17T12:45:27,316 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847527316,5,FailOnTimeoutGroup] 2024-11-17T12:45:27,316 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,317 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T12:45:27,318 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,318 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741831_1007 (size=1321) 2024-11-17T12:45:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741831_1007 (size=1321) 2024-11-17T12:45:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741831_1007 (size=1321) 2024-11-17T12:45:27,322 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T12:45:27,323 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8 2024-11-17T12:45:27,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741832_1008 (size=32) 2024-11-17T12:45:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741832_1008 (size=32) 2024-11-17T12:45:27,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741832_1008 (size=32) 2024-11-17T12:45:27,339 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:27,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T12:45:27,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T12:45:27,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:27,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:27,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T12:45:27,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T12:45:27,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:27,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:27,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T12:45:27,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T12:45:27,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:27,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:27,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T12:45:27,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T12:45:27,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:27,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:27,358 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T12:45:27,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740 2024-11-17T12:45:27,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740 2024-11-17T12:45:27,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T12:45:27,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T12:45:27,363 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T12:45:27,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T12:45:27,369 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:27,370 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61365950, jitterRate=-0.08557608723640442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T12:45:27,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731847527340Initializing all the Stores at 1731847527341 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847527341Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847527341Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847527341Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847527341Cleaning up temporary data from old regions at 1731847527362 (+21 ms)Region opened successfully at 1731847527374 (+12 ms) 2024-11-17T12:45:27,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T12:45:27,374 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T12:45:27,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T12:45:27,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T12:45:27,374 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T12:45:27,376 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T12:45:27,376 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731847527374Disabling compacts and flushes for region at 1731847527374Disabling writes for close at 1731847527374Writing region close event to WAL at 1731847527375 (+1 ms)Closed at 1731847527376 (+1 ms) 2024-11-17T12:45:27,379 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T12:45:27,379 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T12:45:27,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T12:45:27,394 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T12:45:27,397 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,38549,1731847525223 with port=32849, startcode=1731847526132 2024-11-17T12:45:27,397 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,38549,1731847525223 with port=38707, startcode=1731847525964 2024-11-17T12:45:27,397 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(2659): reportForDuty to master=0d751a622496,38549,1731847525223 with port=44359, startcode=1731847526081 2024-11-17T12:45:27,398 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T12:45:27,398 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,44359,1731847526081 2024-11-17T12:45:27,400 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] master.ServerManager(517): Registering regionserver=0d751a622496,44359,1731847526081 2024-11-17T12:45:27,407 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,32849,1731847526132 2024-11-17T12:45:27,408 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] master.ServerManager(517): Registering regionserver=0d751a622496,32849,1731847526132 2024-11-17T12:45:27,408 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8 2024-11-17T12:45:27,408 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37973 2024-11-17T12:45:27,408 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T12:45:27,411 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0d751a622496,38707,1731847525964 2024-11-17T12:45:27,411 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8 2024-11-17T12:45:27,411 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37973 2024-11-17T12:45:27,411 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] master.ServerManager(517): Registering regionserver=0d751a622496,38707,1731847525964 2024-11-17T12:45:27,411 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T12:45:27,416 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8 2024-11-17T12:45:27,416 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37973 2024-11-17T12:45:27,416 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T12:45:27,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T12:45:27,468 DEBUG [RS:1;0d751a622496:44359 {}] zookeeper.ZKUtil(111): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,44359,1731847526081 2024-11-17T12:45:27,469 WARN [RS:1;0d751a622496:44359 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T12:45:27,469 INFO [RS:1;0d751a622496:44359 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:27,469 DEBUG [RS:2;0d751a622496:32849 {}] zookeeper.ZKUtil(111): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,32849,1731847526132 2024-11-17T12:45:27,469 DEBUG [RS:0;0d751a622496:38707 {}] zookeeper.ZKUtil(111): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0d751a622496,38707,1731847525964 2024-11-17T12:45:27,469 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081 2024-11-17T12:45:27,469 WARN [RS:2;0d751a622496:32849 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T12:45:27,469 WARN [RS:0;0d751a622496:38707 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T12:45:27,469 INFO [RS:0;0d751a622496:38707 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:27,469 INFO [RS:2;0d751a622496:32849 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:27,470 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,38707,1731847525964 2024-11-17T12:45:27,470 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,32849,1731847526132 2024-11-17T12:45:27,470 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,32849,1731847526132] 2024-11-17T12:45:27,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,38707,1731847525964] 2024-11-17T12:45:27,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0d751a622496,44359,1731847526081] 2024-11-17T12:45:27,498 INFO [RS:1;0d751a622496:44359 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T12:45:27,498 INFO [RS:2;0d751a622496:32849 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T12:45:27,498 INFO [RS:0;0d751a622496:38707 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T12:45:27,514 INFO [RS:2;0d751a622496:32849 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T12:45:27,514 INFO [RS:0;0d751a622496:38707 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T12:45:27,514 INFO [RS:1;0d751a622496:44359 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T12:45:27,519 INFO [RS:1;0d751a622496:44359 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T12:45:27,519 INFO [RS:2;0d751a622496:32849 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T12:45:27,519 INFO [RS:0;0d751a622496:38707 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T12:45:27,519 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,519 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,519 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,520 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T12:45:27,520 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T12:45:27,520 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T12:45:27,525 INFO [RS:1;0d751a622496:44359 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T12:45:27,525 INFO [RS:2;0d751a622496:32849 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T12:45:27,525 INFO [RS:0;0d751a622496:38707 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T12:45:27,527 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,527 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,527 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,527 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,527 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2 2024-11-17T12:45:27,528 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2 2024-11-17T12:45:27,528 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0, corePoolSize=2, maxPoolSize=2 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3 2024-11-17T12:45:27,528 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,528 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,529 DEBUG [RS:1;0d751a622496:44359 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3 2024-11-17T12:45:27,529 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,529 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,529 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,529 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0d751a622496:0, corePoolSize=1, maxPoolSize=1 2024-11-17T12:45:27,529 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3 2024-11-17T12:45:27,529 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3 2024-11-17T12:45:27,529 DEBUG [RS:2;0d751a622496:32849 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3 2024-11-17T12:45:27,529 DEBUG [RS:0;0d751a622496:38707 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0d751a622496:0, corePoolSize=3, maxPoolSize=3 2024-11-17T12:45:27,534 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,534 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,534 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,534 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,534 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,44359,1731847526081-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38707,1731847525964-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,535 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,536 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,536 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,536 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,536 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,32849,1731847526132-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T12:45:27,549 WARN [0d751a622496:38549 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T12:45:27,555 INFO [RS:0;0d751a622496:38707 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T12:45:27,557 INFO [RS:1;0d751a622496:44359 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T12:45:27,557 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38707,1731847525964-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,558 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,44359,1731847526081-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,558 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,558 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,558 INFO [RS:0;0d751a622496:38707 {}] regionserver.Replication(171): 0d751a622496,38707,1731847525964 started 2024-11-17T12:45:27,558 INFO [RS:1;0d751a622496:44359 {}] regionserver.Replication(171): 0d751a622496,44359,1731847526081 started 2024-11-17T12:45:27,559 INFO [RS:2;0d751a622496:32849 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T12:45:27,559 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,32849,1731847526132-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,559 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,559 INFO [RS:2;0d751a622496:32849 {}] regionserver.Replication(171): 0d751a622496,32849,1731847526132 started 2024-11-17T12:45:27,579 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,579 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,38707,1731847525964, RpcServer on 0d751a622496/172.17.0.2:38707, sessionid=0x1014a1da31f0001 2024-11-17T12:45:27,580 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T12:45:27,580 DEBUG [RS:0;0d751a622496:38707 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,38707,1731847525964 2024-11-17T12:45:27,580 DEBUG [RS:0;0d751a622496:38707 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,38707,1731847525964' 2024-11-17T12:45:27,581 DEBUG [RS:0;0d751a622496:38707 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T12:45:27,581 DEBUG [RS:0;0d751a622496:38707 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T12:45:27,582 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,582 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:27,582 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,32849,1731847526132, RpcServer on 0d751a622496/172.17.0.2:32849, sessionid=0x1014a1da31f0003 2024-11-17T12:45:27,582 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1482): Serving as 0d751a622496,44359,1731847526081, RpcServer on 0d751a622496/172.17.0.2:44359, sessionid=0x1014a1da31f0002 2024-11-17T12:45:27,582 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T12:45:27,582 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T12:45:27,582 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T12:45:27,582 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T12:45:27,582 DEBUG [RS:2;0d751a622496:32849 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,32849,1731847526132 2024-11-17T12:45:27,582 DEBUG [RS:1;0d751a622496:44359 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0d751a622496,44359,1731847526081 2024-11-17T12:45:27,582 DEBUG [RS:0;0d751a622496:38707 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,38707,1731847525964 2024-11-17T12:45:27,582 DEBUG [RS:2;0d751a622496:32849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,32849,1731847526132' 2024-11-17T12:45:27,582 DEBUG [RS:1;0d751a622496:44359 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,44359,1731847526081' 2024-11-17T12:45:27,582 DEBUG [RS:0;0d751a622496:38707 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,38707,1731847525964' 2024-11-17T12:45:27,583 DEBUG [RS:0;0d751a622496:38707 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T12:45:27,583 DEBUG [RS:2;0d751a622496:32849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T12:45:27,583 DEBUG [RS:1;0d751a622496:44359 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T12:45:27,583 DEBUG [RS:0;0d751a622496:38707 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T12:45:27,583 DEBUG [RS:2;0d751a622496:32849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T12:45:27,583 DEBUG [RS:1;0d751a622496:44359 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T12:45:27,584 DEBUG [RS:0;0d751a622496:38707 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T12:45:27,584 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T12:45:27,584 INFO [RS:0;0d751a622496:38707 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T12:45:27,584 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T12:45:27,584 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T12:45:27,584 INFO [RS:0;0d751a622496:38707 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T12:45:27,584 DEBUG [RS:2;0d751a622496:32849 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,32849,1731847526132 2024-11-17T12:45:27,584 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T12:45:27,584 DEBUG [RS:2;0d751a622496:32849 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,32849,1731847526132' 2024-11-17T12:45:27,584 DEBUG [RS:1;0d751a622496:44359 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0d751a622496,44359,1731847526081 2024-11-17T12:45:27,584 DEBUG [RS:2;0d751a622496:32849 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T12:45:27,584 DEBUG [RS:1;0d751a622496:44359 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0d751a622496,44359,1731847526081' 2024-11-17T12:45:27,584 DEBUG [RS:1;0d751a622496:44359 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T12:45:27,585 DEBUG [RS:2;0d751a622496:32849 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T12:45:27,585 DEBUG [RS:1;0d751a622496:44359 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T12:45:27,585 DEBUG [RS:2;0d751a622496:32849 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T12:45:27,585 DEBUG [RS:1;0d751a622496:44359 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T12:45:27,585 INFO [RS:1;0d751a622496:44359 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T12:45:27,585 INFO [RS:2;0d751a622496:32849 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T12:45:27,585 INFO [RS:1;0d751a622496:44359 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T12:45:27,585 INFO [RS:2;0d751a622496:32849 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T12:45:27,690 INFO [RS:0;0d751a622496:38707 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:27,690 INFO [RS:1;0d751a622496:44359 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:27,690 INFO [RS:2;0d751a622496:32849 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:27,693 INFO [RS:1;0d751a622496:44359 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C44359%2C1731847526081, suffix=, logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081, archiveDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs, maxLogs=32 2024-11-17T12:45:27,693 INFO [RS:0;0d751a622496:38707 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C38707%2C1731847525964, suffix=, logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,38707,1731847525964, archiveDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs, maxLogs=32 2024-11-17T12:45:27,693 INFO [RS:2;0d751a622496:32849 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C32849%2C1731847526132, suffix=, logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,32849,1731847526132, archiveDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs, maxLogs=32 2024-11-17T12:45:27,710 DEBUG [RS:1;0d751a622496:44359 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081/0d751a622496%2C44359%2C1731847526081.1731847527698, exclude list is [], retry=0 2024-11-17T12:45:27,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:27,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:27,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:27,717 DEBUG [RS:2;0d751a622496:32849 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,32849,1731847526132/0d751a622496%2C32849%2C1731847526132.1731847527698, exclude list is [], retry=0 2024-11-17T12:45:27,717 DEBUG [RS:0;0d751a622496:38707 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,38707,1731847525964/0d751a622496%2C38707%2C1731847525964.1731847527698, exclude list is [], retry=0 2024-11-17T12:45:27,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:27,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:27,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:27,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:27,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:27,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:27,747 INFO [RS:1;0d751a622496:44359 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081/0d751a622496%2C44359%2C1731847526081.1731847527698 2024-11-17T12:45:27,748 DEBUG [RS:1;0d751a622496:44359 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:27,761 INFO [RS:0;0d751a622496:38707 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,38707,1731847525964/0d751a622496%2C38707%2C1731847525964.1731847527698 2024-11-17T12:45:27,761 INFO [RS:2;0d751a622496:32849 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,32849,1731847526132/0d751a622496%2C32849%2C1731847526132.1731847527698 2024-11-17T12:45:27,763 DEBUG [RS:0;0d751a622496:38707 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:27,763 DEBUG [RS:2;0d751a622496:32849 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:27,801 DEBUG [0d751a622496:38549 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-17T12:45:27,807 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(204): Hosts are {0d751a622496=0} racks are {/default-rack=0} 2024-11-17T12:45:27,814 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T12:45:27,814 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T12:45:27,814 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-17T12:45:27,814 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T12:45:27,814 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T12:45:27,814 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-17T12:45:27,814 INFO [0d751a622496:38549 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T12:45:27,814 INFO [0d751a622496:38549 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T12:45:27,814 INFO [0d751a622496:38549 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-17T12:45:27,814 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T12:45:27,820 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,44359,1731847526081 2024-11-17T12:45:27,826 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,44359,1731847526081, state=OPENING 2024-11-17T12:45:27,875 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T12:45:27,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:45:27,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:27,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:27,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:27,884 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:27,885 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T12:45:27,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,44359,1731847526081}] 2024-11-17T12:45:28,068 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T12:45:28,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33389, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T12:45:28,083 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T12:45:28,083 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:28,084 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T12:45:28,088 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C44359%2C1731847526081.meta, suffix=.meta, logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081, archiveDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs, maxLogs=32 2024-11-17T12:45:28,104 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, exclude list is [], retry=0 2024-11-17T12:45:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:28,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:28,134 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta 2024-11-17T12:45:28,135 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:28,135 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:28,137 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T12:45:28,139 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T12:45:28,144 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T12:45:28,150 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T12:45:28,150 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:28,151 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T12:45:28,151 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T12:45:28,154 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T12:45:28,156 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T12:45:28,156 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:28,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:28,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T12:45:28,159 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T12:45:28,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:28,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:28,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T12:45:28,162 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T12:45:28,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:28,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:28,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T12:45:28,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T12:45:28,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:28,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:28,167 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T12:45:28,168 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740 2024-11-17T12:45:28,172 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740 2024-11-17T12:45:28,175 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T12:45:28,175 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T12:45:28,176 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T12:45:28,178 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T12:45:28,180 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67894296, jitterRate=0.011703848838806152}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T12:45:28,181 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T12:45:28,182 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731847528151Writing region info on filesystem at 1731847528152 (+1 ms)Initializing all the Stores at 1731847528154 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847528154Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847528154Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847528154Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847528154Cleaning up temporary data from old regions at 1731847528175 (+21 ms)Running coprocessor post-open hooks at 1731847528181 (+6 ms)Region opened successfully at 1731847528182 (+1 ms) 2024-11-17T12:45:28,189 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731847528061 2024-11-17T12:45:28,201 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T12:45:28,202 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T12:45:28,204 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,44359,1731847526081 2024-11-17T12:45:28,206 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,44359,1731847526081, state=OPEN 2024-11-17T12:45:28,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:28,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:28,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:28,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:28,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:28,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:28,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:28,240 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:28,241 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,44359,1731847526081 2024-11-17T12:45:28,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T12:45:28,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,44359,1731847526081 in 355 msec 2024-11-17T12:45:28,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T12:45:28,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 867 msec 2024-11-17T12:45:28,259 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T12:45:28,259 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T12:45:28,298 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:28,300 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1] 2024-11-17T12:45:28,327 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T12:45:28,330 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43743, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T12:45:28,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1630 sec 2024-11-17T12:45:28,349 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731847528349, completionTime=-1 2024-11-17T12:45:28,352 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-17T12:45:28,352 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T12:45:28,378 INFO [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-17T12:45:28,378 INFO [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731847588378 2024-11-17T12:45:28,379 INFO [master/0d751a622496:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731847648379 2024-11-17T12:45:28,379 INFO [master/0d751a622496:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-17T12:45:28,380 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-17T12:45:28,386 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38549,1731847525223-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:28,387 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38549,1731847525223-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:28,387 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38549,1731847525223-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:28,388 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0d751a622496:38549, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:28,389 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:28,389 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:28,395 DEBUG [master/0d751a622496:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T12:45:28,422 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.195sec 2024-11-17T12:45:28,423 INFO [master/0d751a622496:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T12:45:28,425 INFO [master/0d751a622496:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T12:45:28,425 INFO [master/0d751a622496:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T12:45:28,426 INFO [master/0d751a622496:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T12:45:28,426 INFO [master/0d751a622496:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T12:45:28,427 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38549,1731847525223-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T12:45:28,427 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38549,1731847525223-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T12:45:28,432 DEBUG [master/0d751a622496:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T12:45:28,433 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T12:45:28,433 INFO [master/0d751a622496:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0d751a622496,38549,1731847525223-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:28,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30246803, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T12:45:28,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0d751a622496,38549,-1 for getting cluster id 2024-11-17T12:45:28,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T12:45:28,503 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e64d9651-704a-4260-becd-b88a4609dc90' 2024-11-17T12:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T12:45:28,506 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e64d9651-704a-4260-becd-b88a4609dc90" 2024-11-17T12:45:28,507 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1666388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T12:45:28,507 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0d751a622496,38549,-1] 2024-11-17T12:45:28,510 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T12:45:28,512 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:45:28,513 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T12:45:28,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13fc4261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T12:45:28,516 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:28,523 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1] 2024-11-17T12:45:28,523 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T12:45:28,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34660, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T12:45:28,545 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0d751a622496,38549,1731847525223 2024-11-17T12:45:28,546 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:37973/hbase 2024-11-17T12:45:28,563 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=6490 2024-11-17T12:45:28,581 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:28,585 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:28,586 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:28,591 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-03409360, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-03409360, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:28,606 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-03409360/hregion-03409360.1731847528593, exclude list is [], retry=0 2024-11-17T12:45:28,611 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:28,612 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:28,612 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:28,642 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-03409360/hregion-03409360.1731847528593 2024-11-17T12:45:28,643 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:28,643 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c6ec393c212c5032a0a4667f4f1bb0f8, NAME => 'testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:28,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741838_1014 (size=64) 2024-11-17T12:45:28,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741838_1014 (size=64) 2024-11-17T12:45:28,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741838_1014 (size=64) 2024-11-17T12:45:28,658 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:28,660 INFO [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:28,663 INFO [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6ec393c212c5032a0a4667f4f1bb0f8 columnFamilyName a 2024-11-17T12:45:28,663 DEBUG [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:28,664 INFO [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] regionserver.HStore(327): Store=c6ec393c212c5032a0a4667f4f1bb0f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:28,664 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:28,666 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:28,666 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:28,667 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:28,667 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:28,670 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:28,674 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:28,675 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c6ec393c212c5032a0a4667f4f1bb0f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74585837, jitterRate=0.11141557991504669}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:28,676 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c6ec393c212c5032a0a4667f4f1bb0f8: Writing region info on filesystem at 1731847528658Initializing all the Stores at 1731847528659 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847528660 (+1 ms)Cleaning up temporary data from old regions at 1731847528667 (+7 ms)Region opened successfully at 1731847528676 (+9 ms) 2024-11-17T12:45:28,676 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c6ec393c212c5032a0a4667f4f1bb0f8, disabling compactions & flushes 2024-11-17T12:45:28,676 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8. 2024-11-17T12:45:28,676 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8. 2024-11-17T12:45:28,676 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8. after waiting 0 ms 2024-11-17T12:45:28,676 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8. 2024-11-17T12:45:28,677 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8. 2024-11-17T12:45:28,677 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c6ec393c212c5032a0a4667f4f1bb0f8: Waiting for close lock at 1731847528676Disabling compacts and flushes for region at 1731847528676Disabling writes for close at 1731847528676Writing region close event to WAL at 1731847528677 (+1 ms)Closed at 1731847528677 2024-11-17T12:45:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741837_1013 (size=175) 2024-11-17T12:45:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741837_1013 (size=175) 2024-11-17T12:45:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741837_1013 (size=175) 2024-11-17T12:45:28,689 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:28,689 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-03409360:(num 1731847528593) 2024-11-17T12:45:28,691 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-17T12:45:28,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741839_1015 (size=469) 2024-11-17T12:45:28,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741839_1015 (size=469) 2024-11-17T12:45:28,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741839_1015 (size=469) 2024-11-17T12:45:28,736 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-17T12:45:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741840_1016 (size=322) 2024-11-17T12:45:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741840_1016 (size=322) 2024-11-17T12:45:28,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741840_1016 (size=322) 2024-11-17T12:45:28,793 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1, size=469 (469bytes) 2024-11-17T12:45:28,793 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-17T12:45:28,793 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-17T12:45:28,794 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1 2024-11-17T12:45:28,799 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1 after 3ms 2024-11-17T12:45:28,831 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:28,832 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1 took 40ms 2024-11-17T12:45:28,838 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1 so closing down 2024-11-17T12:45:28,838 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:28,841 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-17T12:45:28,867 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000001-wal-1.temp 2024-11-17T12:45:28,869 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:28,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741841_1017 (size=469) 2024-11-17T12:45:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741841_1017 (size=469) 2024-11-17T12:45:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741841_1017 (size=469) 2024-11-17T12:45:28,882 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:28,885 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002 2024-11-17T12:45:28,890 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-17T12:45:28,890 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1, journal: Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1, size=469 (469bytes) at 1731847528793Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1 so closing down at 1731847528838 (+45 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000001-wal-1.temp at 1731847528868 (+30 ms)3 split writer threads finished at 1731847528869 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731847528882 (+13 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002 at 1731847528885 (+3 ms)Processed 2 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731847528890 (+5 ms) 2024-11-17T12:45:28,905 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2, size=322 (322bytes) 2024-11-17T12:45:28,905 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2 2024-11-17T12:45:28,906 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2 after 1ms 2024-11-17T12:45:28,935 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:28,936 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2 took 31ms 2024-11-17T12:45:28,939 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2 so closing down 2024-11-17T12:45:28,939 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:28,942 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-17T12:45:28,966 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002-wal-2.temp 2024-11-17T12:45:28,966 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:28,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741842_1018 (size=322) 2024-11-17T12:45:28,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741842_1018 (size=322) 2024-11-17T12:45:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741842_1018 (size=322) 2024-11-17T12:45:28,976 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:29,005 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:29,008 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-17T12:45:29,010 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-17T12:45:29,010 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2, journal: Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2, size=322 (322bytes) at 1731847528905Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2 so closing down at 1731847528939 (+34 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002-wal-2.temp at 1731847528966 (+27 ms)3 split writer threads finished at 1731847528966Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731847528976 (+10 ms)Processed 1 edits across 1 Regions in 74 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731847529010 (+34 ms) 2024-11-17T12:45:29,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:29,012 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:29,025 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal.1731847529013, exclude list is [], retry=0 2024-11-17T12:45:29,029 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:29,030 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:29,030 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:29,056 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal.1731847529013 2024-11-17T12:45:29,056 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:29,056 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c6ec393c212c5032a0a4667f4f1bb0f8, NAME => 'testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:29,057 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:29,057 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,057 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,059 INFO [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,060 INFO [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c6ec393c212c5032a0a4667f4f1bb0f8 columnFamilyName a 2024-11-17T12:45:29,061 DEBUG [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:29,062 INFO [StoreOpener-c6ec393c212c5032a0a4667f4f1bb0f8-1 {}] regionserver.HStore(327): Store=c6ec393c212c5032a0a4667f4f1bb0f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:29,062 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,063 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,067 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002 2024-11-17T12:45:29,093 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:29,098 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002 2024-11-17T12:45:29,101 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c6ec393c212c5032a0a4667f4f1bb0f8 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-17T12:45:29,153 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/.tmp/a/56a004fbeda743fab85f92c19f05e1f0 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731847528689/Put/seqid=0 2024-11-17T12:45:29,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741844_1020 (size=5170) 2024-11-17T12:45:29,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741844_1020 (size=5170) 2024-11-17T12:45:29,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741844_1020 (size=5170) 2024-11-17T12:45:29,167 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/.tmp/a/56a004fbeda743fab85f92c19f05e1f0 2024-11-17T12:45:29,210 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/.tmp/a/56a004fbeda743fab85f92c19f05e1f0 as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/a/56a004fbeda743fab85f92c19f05e1f0 2024-11-17T12:45:29,221 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/a/56a004fbeda743fab85f92c19f05e1f0, entries=2, sequenceid=2, filesize=5.0 K 2024-11-17T12:45:29,229 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c6ec393c212c5032a0a4667f4f1bb0f8 in 124ms, sequenceid=2, compaction requested=false; wal=null 2024-11-17T12:45:29,231 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/0000000000000000002 2024-11-17T12:45:29,232 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,232 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,238 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c6ec393c212c5032a0a4667f4f1bb0f8 2024-11-17T12:45:29,242 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/c6ec393c212c5032a0a4667f4f1bb0f8/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-17T12:45:29,244 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c6ec393c212c5032a0a4667f4f1bb0f8; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70659506, jitterRate=0.05290868878364563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:29,246 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c6ec393c212c5032a0a4667f4f1bb0f8: Writing region info on filesystem at 1731847529057Initializing all the Stores at 1731847529058 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847529058Obtaining lock to block concurrent updates at 1731847529101 (+43 ms)Preparing flush snapshotting stores in c6ec393c212c5032a0a4667f4f1bb0f8 at 1731847529101Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731847529104 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731847528582.c6ec393c212c5032a0a4667f4f1bb0f8. at 1731847529104Flushing c6ec393c212c5032a0a4667f4f1bb0f8/a: creating writer at 1731847529105 (+1 ms)Flushing c6ec393c212c5032a0a4667f4f1bb0f8/a: appending metadata at 1731847529142 (+37 ms)Flushing c6ec393c212c5032a0a4667f4f1bb0f8/a: closing flushed file at 1731847529145 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ebdd5e7: reopening flushed file at 1731847529208 (+63 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for c6ec393c212c5032a0a4667f4f1bb0f8 in 124ms, sequenceid=2, compaction requested=false; wal=null at 1731847529229 (+21 ms)Cleaning up temporary data from old regions at 1731847529232 (+3 ms)Region opened successfully at 1731847529246 (+14 ms) 2024-11-17T12:45:29,279 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=375 (was 364) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:56578 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:44082 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:35452 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:44242 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:56446 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:35558 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:37973/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=6473 (was 6490) 2024-11-17T12:45:29,290 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=6472 2024-11-17T12:45:29,304 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:29,307 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:29,308 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:29,312 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95565493, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-95565493, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:29,326 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95565493/hregion-95565493.1731847529312, exclude list is [], retry=0 2024-11-17T12:45:29,330 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:29,330 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:29,331 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:29,359 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95565493/hregion-95565493.1731847529312 2024-11-17T12:45:29,359 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:29,359 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 06fd419e4b14f8a8d7ccbee0bcf9faf7, NAME => 'testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:29,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741846_1022 (size=64) 2024-11-17T12:45:29,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741846_1022 (size=64) 2024-11-17T12:45:29,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741846_1022 (size=64) 2024-11-17T12:45:29,374 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:29,376 INFO [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,378 INFO [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06fd419e4b14f8a8d7ccbee0bcf9faf7 columnFamilyName a 2024-11-17T12:45:29,378 DEBUG [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:29,379 INFO [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] regionserver.HStore(327): Store=06fd419e4b14f8a8d7ccbee0bcf9faf7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:29,379 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,380 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,381 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,382 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,382 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,384 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,387 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:29,388 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06fd419e4b14f8a8d7ccbee0bcf9faf7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73641654, jitterRate=0.09734615683555603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:29,388 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06fd419e4b14f8a8d7ccbee0bcf9faf7: Writing region info on filesystem at 1731847529374Initializing all the Stores at 1731847529376 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847529376Cleaning up temporary data from old regions at 1731847529382 (+6 ms)Region opened successfully at 1731847529388 (+6 ms) 2024-11-17T12:45:29,388 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 06fd419e4b14f8a8d7ccbee0bcf9faf7, disabling compactions & flushes 2024-11-17T12:45:29,388 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7. 2024-11-17T12:45:29,388 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7. 2024-11-17T12:45:29,388 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7. after waiting 0 ms 2024-11-17T12:45:29,388 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7. 2024-11-17T12:45:29,389 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7. 2024-11-17T12:45:29,389 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 06fd419e4b14f8a8d7ccbee0bcf9faf7: Waiting for close lock at 1731847529388Disabling compacts and flushes for region at 1731847529388Disabling writes for close at 1731847529388Writing region close event to WAL at 1731847529389 (+1 ms)Closed at 1731847529389 2024-11-17T12:45:29,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741845_1021 (size=175) 2024-11-17T12:45:29,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741845_1021 (size=175) 2024-11-17T12:45:29,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741845_1021 (size=175) 2024-11-17T12:45:29,398 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:29,398 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95565493:(num 1731847529312) 2024-11-17T12:45:29,399 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-17T12:45:29,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741847_1023 (size=469) 2024-11-17T12:45:29,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741847_1023 (size=469) 2024-11-17T12:45:29,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741847_1023 (size=469) 2024-11-17T12:45:29,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-17T12:45:29,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741848_1024 (size=322) 2024-11-17T12:45:29,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741848_1024 (size=322) 2024-11-17T12:45:29,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741848_1024 (size=322) 2024-11-17T12:45:29,488 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2, size=322 (322bytes) 2024-11-17T12:45:29,488 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2 2024-11-17T12:45:29,489 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2 after 1ms 2024-11-17T12:45:29,514 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:29,514 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2 took 26ms 2024-11-17T12:45:29,517 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2 so closing down 2024-11-17T12:45:29,517 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:29,519 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-17T12:45:29,545 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002-wal-2.temp 2024-11-17T12:45:29,546 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:29,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741849_1025 (size=322) 2024-11-17T12:45:29,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741849_1025 (size=322) 2024-11-17T12:45:29,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741849_1025 (size=322) 2024-11-17T12:45:29,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:29,557 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002 2024-11-17T12:45:29,557 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-17T12:45:29,557 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2, journal: Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2, size=322 (322bytes) at 1731847529488Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2 so closing down at 1731847529517 (+29 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002-wal-2.temp at 1731847529546 (+29 ms)3 split writer threads finished at 1731847529546Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731847529555 (+9 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002 at 1731847529557 (+2 ms)Processed 1 edits across 1 Regions in 42 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731847529557 2024-11-17T12:45:29,573 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1, size=469 (469bytes) 2024-11-17T12:45:29,573 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1 2024-11-17T12:45:29,573 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1 after 0ms 2024-11-17T12:45:29,599 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:29,599 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1 took 26ms 2024-11-17T12:45:29,603 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1 so closing down 2024-11-17T12:45:29,603 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:29,605 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-17T12:45:29,629 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000001-wal-1.temp 2024-11-17T12:45:29,629 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:29,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741850_1026 (size=469) 2024-11-17T12:45:29,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741850_1026 (size=469) 2024-11-17T12:45:29,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741850_1026 (size=469) 2024-11-17T12:45:29,639 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:29,666 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:29,668 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002, length=322 2024-11-17T12:45:29,670 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002 2024-11-17T12:45:29,670 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-17T12:45:29,671 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1, journal: Splitting hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1, size=469 (469bytes) at 1731847529573Finishing writing output for hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1 so closing down at 1731847529603 (+30 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000001-wal-1.temp at 1731847529629 (+26 ms)3 split writer threads finished at 1731847529630 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731847529639 (+9 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002 at 1731847529670 (+31 ms)Processed 2 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731847529671 (+1 ms) 2024-11-17T12:45:29,671 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:29,673 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:29,687 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal.1731847529674, exclude list is [], retry=0 2024-11-17T12:45:29,690 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:29,691 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:29,691 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:29,714 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal.1731847529674 2024-11-17T12:45:29,714 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:29,714 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 06fd419e4b14f8a8d7ccbee0bcf9faf7, NAME => 'testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:29,715 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:29,715 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,715 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,717 INFO [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,719 INFO [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 06fd419e4b14f8a8d7ccbee0bcf9faf7 columnFamilyName a 2024-11-17T12:45:29,719 DEBUG [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:29,719 INFO [StoreOpener-06fd419e4b14f8a8d7ccbee0bcf9faf7-1 {}] regionserver.HStore(327): Store=06fd419e4b14f8a8d7ccbee0bcf9faf7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:29,720 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,720 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,722 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,723 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002 2024-11-17T12:45:29,751 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:29,753 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002 2024-11-17T12:45:29,753 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 06fd419e4b14f8a8d7ccbee0bcf9faf7 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-17T12:45:29,767 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/.tmp/a/2ba4d94f6d404ba493edda343054fc1e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731847529398/Put/seqid=0 2024-11-17T12:45:29,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741852_1028 (size=5170) 2024-11-17T12:45:29,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741852_1028 (size=5170) 2024-11-17T12:45:29,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741852_1028 (size=5170) 2024-11-17T12:45:29,776 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/.tmp/a/2ba4d94f6d404ba493edda343054fc1e 2024-11-17T12:45:29,785 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/.tmp/a/2ba4d94f6d404ba493edda343054fc1e as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/a/2ba4d94f6d404ba493edda343054fc1e 2024-11-17T12:45:29,793 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/a/2ba4d94f6d404ba493edda343054fc1e, entries=2, sequenceid=2, filesize=5.0 K 2024-11-17T12:45:29,794 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 06fd419e4b14f8a8d7ccbee0bcf9faf7 in 40ms, sequenceid=2, compaction requested=false; wal=null 2024-11-17T12:45:29,794 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/0000000000000000002 2024-11-17T12:45:29,795 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,795 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,798 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 06fd419e4b14f8a8d7ccbee0bcf9faf7 2024-11-17T12:45:29,801 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/06fd419e4b14f8a8d7ccbee0bcf9faf7/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-17T12:45:29,802 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 06fd419e4b14f8a8d7ccbee0bcf9faf7; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69215778, jitterRate=0.03139546513557434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:29,802 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 06fd419e4b14f8a8d7ccbee0bcf9faf7: Writing region info on filesystem at 1731847529715Initializing all the Stores at 1731847529716 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847529716Obtaining lock to block concurrent updates at 1731847529753 (+37 ms)Preparing flush snapshotting stores in 06fd419e4b14f8a8d7ccbee0bcf9faf7 at 1731847529753Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731847529753Flushing stores of testReplayEditsWrittenIntoWAL,,1731847529305.06fd419e4b14f8a8d7ccbee0bcf9faf7. at 1731847529753Flushing 06fd419e4b14f8a8d7ccbee0bcf9faf7/a: creating writer at 1731847529753Flushing 06fd419e4b14f8a8d7ccbee0bcf9faf7/a: appending metadata at 1731847529766 (+13 ms)Flushing 06fd419e4b14f8a8d7ccbee0bcf9faf7/a: closing flushed file at 1731847529766Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12237b68: reopening flushed file at 1731847529784 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 06fd419e4b14f8a8d7ccbee0bcf9faf7 in 40ms, sequenceid=2, compaction requested=false; wal=null at 1731847529794 (+10 ms)Cleaning up temporary data from old regions at 1731847529795 (+1 ms)Region opened successfully at 1731847529802 (+7 ms) 2024-11-17T12:45:29,819 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:44082 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:35644 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:44304 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:35452 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:56446 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:56630 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=285 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=6470 (was 6472) 2024-11-17T12:45:29,829 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=285, ProcessCount=11, AvailableMemoryMB=6470 2024-11-17T12:45:29,844 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:29,846 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:29,847 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:29,849 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-71733936, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-71733936, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:29,861 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-71733936/hregion-71733936.1731847529850, exclude list is [], retry=0 2024-11-17T12:45:29,865 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:29,866 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:29,866 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:29,893 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-71733936/hregion-71733936.1731847529850 2024-11-17T12:45:29,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:29,894 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1e6174890ebf7134ac309eb8070b1581, NAME => 'testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:29,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741854_1030 (size=64) 2024-11-17T12:45:29,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741854_1030 (size=64) 2024-11-17T12:45:29,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741854_1030 (size=64) 2024-11-17T12:45:29,908 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:29,910 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,912 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e6174890ebf7134ac309eb8070b1581 columnFamilyName a 2024-11-17T12:45:29,912 DEBUG [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:29,913 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(327): Store=1e6174890ebf7134ac309eb8070b1581/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:29,913 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,914 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e6174890ebf7134ac309eb8070b1581 columnFamilyName b 2024-11-17T12:45:29,915 DEBUG [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:29,915 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(327): Store=1e6174890ebf7134ac309eb8070b1581/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:29,915 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,917 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e6174890ebf7134ac309eb8070b1581 columnFamilyName c 2024-11-17T12:45:29,917 DEBUG [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:29,918 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(327): Store=1e6174890ebf7134ac309eb8070b1581/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:29,918 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,919 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,920 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,921 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,921 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,922 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:29,923 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:29,926 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:29,926 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e6174890ebf7134ac309eb8070b1581; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64148445, jitterRate=-0.044113680720329285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:29,927 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e6174890ebf7134ac309eb8070b1581: Writing region info on filesystem at 1731847529909Initializing all the Stores at 1731847529910 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847529910Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847529910Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847529910Cleaning up temporary data from old regions at 1731847529921 (+11 ms)Region opened successfully at 1731847529926 (+5 ms) 2024-11-17T12:45:29,927 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1e6174890ebf7134ac309eb8070b1581, disabling compactions & flushes 2024-11-17T12:45:29,927 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:29,927 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:29,927 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. after waiting 0 ms 2024-11-17T12:45:29,927 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:29,927 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:29,927 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1e6174890ebf7134ac309eb8070b1581: Waiting for close lock at 1731847529927Disabling compacts and flushes for region at 1731847529927Disabling writes for close at 1731847529927Writing region close event to WAL at 1731847529927Closed at 1731847529927 2024-11-17T12:45:29,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741853_1029 (size=175) 2024-11-17T12:45:29,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741853_1029 (size=175) 2024-11-17T12:45:29,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741853_1029 (size=175) 2024-11-17T12:45:29,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:29,933 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-71733936:(num 1731847529850) 2024-11-17T12:45:29,933 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:29,935 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:29,949 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936, exclude list is [], retry=0 2024-11-17T12:45:29,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:29,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:29,954 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:29,981 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 2024-11-17T12:45:29,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:30,188 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936, size=0 (0bytes) 2024-11-17T12:45:30,188 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 might be still open, length is 0 2024-11-17T12:45:30,188 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 2024-11-17T12:45:30,189 WARN [IPC Server handler 2 on default port 37973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-17T12:45:30,189 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 after 1ms 2024-11-17T12:45:30,288 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:56668 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56668 dst: /127.0.0.1:44483 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:44483 remote=/127.0.0.1:56668]. Total timeout mills is 60000, 59869 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:30,290 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:35684 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34421:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35684 dst: /127.0.0.1:34421 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:30,290 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:44346 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42393:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44346 dst: /127.0.0.1:42393 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:30,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741855_1032 (size=497775) 2024-11-17T12:45:30,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741855_1032 (size=497775) 2024-11-17T12:45:30,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741855_1032 (size=497775) 2024-11-17T12:45:33,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T12:45:33,720 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T12:45:34,191 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 after 4003ms 2024-11-17T12:45:34,221 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:34,223 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 took 4035ms 2024-11-17T12:45:34,227 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731847529936.temp 2024-11-17T12:45:34,259 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000000001-wal.1731847529936.temp 2024-11-17T12:45:34,325 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936; continuing. 2024-11-17T12:45:34,325 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 so closing down 2024-11-17T12:45:34,325 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:34,325 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:34,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741856_1033 (size=497783) 2024-11-17T12:45:34,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741856_1033 (size=497783) 2024-11-17T12:45:34,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741856_1033 (size=497783) 2024-11-17T12:45:34,329 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000000001-wal.1731847529936.temp (wrote 3002 edits, skipped 0 edits in 45 ms) 2024-11-17T12:45:34,331 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000000001-wal.1731847529936.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000003002 2024-11-17T12:45:34,331 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 108 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936, size=0, length=0, corrupted=false, cancelled=false 2024-11-17T12:45:34,331 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936, journal: Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936, size=0 (0bytes) at 1731847530188Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000000001-wal.1731847529936.temp at 1731847534260 (+4072 ms)Split 1024 edits, skipped 0 edits. at 1731847534290 (+30 ms)Split 2048 edits, skipped 0 edits. at 1731847534311 (+21 ms)Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 so closing down at 1731847534325 (+14 ms)3 split writer threads finished at 1731847534325Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000000001-wal.1731847529936.temp (wrote 3002 edits, skipped 0 edits in 45 ms) at 1731847534329 (+4 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000000001-wal.1731847529936.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000003002 at 1731847534331 (+2 ms)Processed 3002 edits across 1 Regions in 108 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936, size=0, length=0, corrupted=false, cancelled=false at 1731847534331 2024-11-17T12:45:34,334 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847529936 2024-11-17T12:45:34,335 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000003002 2024-11-17T12:45:34,335 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:34,338 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:34,351 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847534338, exclude list is [], retry=0 2024-11-17T12:45:34,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:34,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:34,355 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:34,379 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847534338 2024-11-17T12:45:34,380 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:34,380 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:34,382 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:34,383 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e6174890ebf7134ac309eb8070b1581 columnFamilyName a 2024-11-17T12:45:34,384 DEBUG [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:34,384 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(327): Store=1e6174890ebf7134ac309eb8070b1581/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:34,384 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:34,386 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e6174890ebf7134ac309eb8070b1581 columnFamilyName b 2024-11-17T12:45:34,386 DEBUG [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:34,387 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(327): Store=1e6174890ebf7134ac309eb8070b1581/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:34,387 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:34,388 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e6174890ebf7134ac309eb8070b1581 columnFamilyName c 2024-11-17T12:45:34,388 DEBUG [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:34,389 INFO [StoreOpener-1e6174890ebf7134ac309eb8070b1581-1 {}] regionserver.HStore(327): Store=1e6174890ebf7134ac309eb8070b1581/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:34,389 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:34,390 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:34,393 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:34,394 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000003002 2024-11-17T12:45:34,422 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:34,458 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-17T12:45:34,799 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e6174890ebf7134ac309eb8070b1581 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-17T12:45:34,836 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/a/5b381ec40117476eb60e00a1b9dcf75e is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731847529993/Put/seqid=0 2024-11-17T12:45:34,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741858_1035 (size=50463) 2024-11-17T12:45:34,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741858_1035 (size=50463) 2024-11-17T12:45:34,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741858_1035 (size=50463) 2024-11-17T12:45:34,846 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/a/5b381ec40117476eb60e00a1b9dcf75e 2024-11-17T12:45:34,854 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/a/5b381ec40117476eb60e00a1b9dcf75e as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/a/5b381ec40117476eb60e00a1b9dcf75e 2024-11-17T12:45:34,860 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/a/5b381ec40117476eb60e00a1b9dcf75e, entries=754, sequenceid=754, filesize=49.3 K 2024-11-17T12:45:34,860 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 1e6174890ebf7134ac309eb8070b1581 in 62ms, sequenceid=754, compaction requested=false; wal=null 2024-11-17T12:45:34,876 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-17T12:45:34,876 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e6174890ebf7134ac309eb8070b1581 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-17T12:45:34,887 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/a/911a9fe3a2e24ac09d3b87e8e9ff43cb is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731847530051/Put/seqid=0 2024-11-17T12:45:34,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741859_1036 (size=20072) 2024-11-17T12:45:34,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741859_1036 (size=20072) 2024-11-17T12:45:34,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741859_1036 (size=20072) 2024-11-17T12:45:34,899 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/a/911a9fe3a2e24ac09d3b87e8e9ff43cb 2024-11-17T12:45:34,924 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/b/5f2c08d7a02642b1aafa65e78c2eb627 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731847530072/Put/seqid=0 2024-11-17T12:45:34,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741860_1037 (size=35835) 2024-11-17T12:45:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741860_1037 (size=35835) 2024-11-17T12:45:34,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741860_1037 (size=35835) 2024-11-17T12:45:34,934 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/b/5f2c08d7a02642b1aafa65e78c2eb627 2024-11-17T12:45:34,943 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/a/911a9fe3a2e24ac09d3b87e8e9ff43cb as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/a/911a9fe3a2e24ac09d3b87e8e9ff43cb 2024-11-17T12:45:34,952 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/a/911a9fe3a2e24ac09d3b87e8e9ff43cb, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-17T12:45:34,953 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/b/5f2c08d7a02642b1aafa65e78c2eb627 as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/b/5f2c08d7a02642b1aafa65e78c2eb627 2024-11-17T12:45:34,960 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/b/5f2c08d7a02642b1aafa65e78c2eb627, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-17T12:45:34,961 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1e6174890ebf7134ac309eb8070b1581 in 84ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-17T12:45:34,982 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-17T12:45:34,982 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e6174890ebf7134ac309eb8070b1581 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-17T12:45:34,992 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/b/a494c98b8068408ca6fdcde62f790d1a is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731847530091/Put/seqid=0 2024-11-17T12:45:35,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741861_1038 (size=35082) 2024-11-17T12:45:35,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741861_1038 (size=35082) 2024-11-17T12:45:35,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741861_1038 (size=35082) 2024-11-17T12:45:35,003 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/b/a494c98b8068408ca6fdcde62f790d1a 2024-11-17T12:45:35,031 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/c/95b065e99c5342ddb4be8b7953521a28 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731847530114/Put/seqid=0 2024-11-17T12:45:35,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741862_1039 (size=20825) 2024-11-17T12:45:35,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741862_1039 (size=20825) 2024-11-17T12:45:35,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741862_1039 (size=20825) 2024-11-17T12:45:35,041 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/c/95b065e99c5342ddb4be8b7953521a28 2024-11-17T12:45:35,050 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/b/a494c98b8068408ca6fdcde62f790d1a as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/b/a494c98b8068408ca6fdcde62f790d1a 2024-11-17T12:45:35,059 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/b/a494c98b8068408ca6fdcde62f790d1a, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-17T12:45:35,061 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/c/95b065e99c5342ddb4be8b7953521a28 as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/c/95b065e99c5342ddb4be8b7953521a28 2024-11-17T12:45:35,067 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/c/95b065e99c5342ddb4be8b7953521a28, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-17T12:45:35,068 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1e6174890ebf7134ac309eb8070b1581 in 85ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-17T12:45:35,079 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731847530155/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:35,083 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000003002 2024-11-17T12:45:35,083 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-17T12:45:35,083 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e6174890ebf7134ac309eb8070b1581 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-17T12:45:35,093 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/c/190f1ebca9f643e4ba99154ce573e67b is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731847530120/Put/seqid=0 2024-11-17T12:45:35,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741863_1040 (size=50301) 2024-11-17T12:45:35,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741863_1040 (size=50301) 2024-11-17T12:45:35,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741863_1040 (size=50301) 2024-11-17T12:45:35,103 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/c/190f1ebca9f643e4ba99154ce573e67b 2024-11-17T12:45:35,111 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 190f1ebca9f643e4ba99154ce573e67b 2024-11-17T12:45:35,113 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/.tmp/c/190f1ebca9f643e4ba99154ce573e67b as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/c/190f1ebca9f643e4ba99154ce573e67b 2024-11-17T12:45:35,120 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 190f1ebca9f643e4ba99154ce573e67b 2024-11-17T12:45:35,121 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/c/190f1ebca9f643e4ba99154ce573e67b, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-17T12:45:35,121 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 1e6174890ebf7134ac309eb8070b1581 in 38ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-17T12:45:35,122 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/0000000000000003002 2024-11-17T12:45:35,123 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:35,123 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:35,124 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T12:45:35,126 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e6174890ebf7134ac309eb8070b1581 2024-11-17T12:45:35,128 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenIntoWAL/1e6174890ebf7134ac309eb8070b1581/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-17T12:45:35,130 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e6174890ebf7134ac309eb8070b1581; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61087391, jitterRate=-0.08972693979740143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T12:45:35,130 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e6174890ebf7134ac309eb8070b1581: Writing region info on filesystem at 1731847534380Initializing all the Stores at 1731847534382 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847534382Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847534382Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847534382Cleaning up temporary data from old regions at 1731847535123 (+741 ms)Region opened successfully at 1731847535130 (+7 ms) 2024-11-17T12:45:35,195 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1e6174890ebf7134ac309eb8070b1581, disabling compactions & flushes 2024-11-17T12:45:35,195 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:35,195 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:35,196 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. after waiting 0 ms 2024-11-17T12:45:35,196 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:35,197 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731847529844.1e6174890ebf7134ac309eb8070b1581. 2024-11-17T12:45:35,198 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1e6174890ebf7134ac309eb8070b1581: Waiting for close lock at 1731847535195Disabling compacts and flushes for region at 1731847535195Disabling writes for close at 1731847535196 (+1 ms)Writing region close event to WAL at 1731847535197 (+1 ms)Closed at 1731847535197 2024-11-17T12:45:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741857_1034 (size=175) 2024-11-17T12:45:35,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741857_1034 (size=175) 2024-11-17T12:45:35,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741857_1034 (size=175) 2024-11-17T12:45:35,205 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:35,205 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731847534338) 2024-11-17T12:45:35,220 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 385) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_618722469_22 at /127.0.0.1:40236 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_618722469_22 at /127.0.0.1:40260 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:34587 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_618722469_22 at /127.0.0.1:50004 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34587 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:37973 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39299 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:39299 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:37973 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_618722469_22 at /127.0.0.1:38922 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=262 (was 285), ProcessCount=11 (was 11), AvailableMemoryMB=6363 (was 6470) 2024-11-17T12:45:35,235 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=262, ProcessCount=11, AvailableMemoryMB=6361 2024-11-17T12:45:35,257 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:35,261 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:35,262 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:35,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-66514264, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-66514264, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:35,278 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-66514264/hregion-66514264.1731847535266, exclude list is [], retry=0 2024-11-17T12:45:35,281 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:35,281 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:35,281 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:35,307 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-66514264/hregion-66514264.1731847535266 2024-11-17T12:45:35,308 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:35,309 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6bb0502de7f710f05d2f473083767b48, NAME => 'test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:35,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741865_1042 (size=43) 2024-11-17T12:45:35,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741865_1042 (size=43) 2024-11-17T12:45:35,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741865_1042 (size=43) 2024-11-17T12:45:35,325 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:35,327 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,329 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bb0502de7f710f05d2f473083767b48 columnFamilyName a 2024-11-17T12:45:35,329 DEBUG [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:35,329 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(327): Store=6bb0502de7f710f05d2f473083767b48/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:35,330 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,332 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bb0502de7f710f05d2f473083767b48 columnFamilyName b 2024-11-17T12:45:35,332 DEBUG [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:35,333 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(327): Store=6bb0502de7f710f05d2f473083767b48/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:35,333 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,335 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bb0502de7f710f05d2f473083767b48 columnFamilyName c 2024-11-17T12:45:35,335 DEBUG [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:35,336 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(327): Store=6bb0502de7f710f05d2f473083767b48/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:35,336 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,339 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:35,341 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:35,344 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:35,345 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6bb0502de7f710f05d2f473083767b48; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60729235, jitterRate=-0.09506388008594513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:35,346 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6bb0502de7f710f05d2f473083767b48: Writing region info on filesystem at 1731847535325Initializing all the Stores at 1731847535327 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847535327Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847535327Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847535327Cleaning up temporary data from old regions at 1731847535339 (+12 ms)Region opened successfully at 1731847535346 (+7 ms) 2024-11-17T12:45:35,346 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6bb0502de7f710f05d2f473083767b48, disabling compactions & flushes 2024-11-17T12:45:35,346 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:35,347 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:35,347 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. after waiting 0 ms 2024-11-17T12:45:35,347 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:35,347 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:35,347 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6bb0502de7f710f05d2f473083767b48: Waiting for close lock at 1731847535346Disabling compacts and flushes for region at 1731847535346Disabling writes for close at 1731847535347 (+1 ms)Writing region close event to WAL at 1731847535347Closed at 1731847535347 2024-11-17T12:45:35,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741864_1041 (size=175) 2024-11-17T12:45:35,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741864_1041 (size=175) 2024-11-17T12:45:35,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741864_1041 (size=175) 2024-11-17T12:45:35,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:35,353 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-66514264:(num 1731847535266) 2024-11-17T12:45:35,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:35,355 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:35,368 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356, exclude list is [], retry=0 2024-11-17T12:45:35,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:35,371 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:35,372 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:35,397 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356 2024-11-17T12:45:35,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:35,568 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T12:45:35,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:35,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T12:45:35,571 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T12:45:35,571 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T12:45:35,571 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:35,572 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-17T12:45:35,572 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:35,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741866_1043 (size=371388) 2024-11-17T12:45:35,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741866_1043 (size=371388) 2024-11-17T12:45:35,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741866_1043 (size=371388) 2024-11-17T12:45:35,627 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356, size=362.7 K (371388bytes) 2024-11-17T12:45:35,627 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356 2024-11-17T12:45:35,628 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356 after 0ms 2024-11-17T12:45:35,677 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:35,679 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356 took 52ms 2024-11-17T12:45:35,688 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731847535356.temp 2024-11-17T12:45:35,728 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356 so closing down 2024-11-17T12:45:35,728 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:35,737 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000000001-wal.1731847535356.temp 2024-11-17T12:45:35,759 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:35,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741867_1044 (size=371388) 2024-11-17T12:45:35,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741867_1044 (size=371388) 2024-11-17T12:45:35,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741867_1044 (size=371388) 2024-11-17T12:45:35,764 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000000001-wal.1731847535356.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-17T12:45:35,767 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000000001-wal.1731847535356.temp to hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003000 2024-11-17T12:45:35,767 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-17T12:45:35,767 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356, journal: Splitting hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356, size=362.7 K (371388bytes) at 1731847535627Split 1024 edits, skipped 0 edits. at 1731847535697 (+70 ms)Split 2048 edits, skipped 0 edits. at 1731847535712 (+15 ms)Finishing writing output for hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356 so closing down at 1731847535728 (+16 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000000001-wal.1731847535356.temp at 1731847535737 (+9 ms)3 split writer threads finished at 1731847535759 (+22 ms)Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000000001-wal.1731847535356.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1731847535764 (+5 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000000001-wal.1731847535356.temp to hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003000 at 1731847535767 (+3 ms)Processed 3000 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731847535767 2024-11-17T12:45:35,769 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535356 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847535356 2024-11-17T12:45:35,771 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003000 2024-11-17T12:45:35,771 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:35,773 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:35,792 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774, exclude list is [], retry=0 2024-11-17T12:45:35,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:35,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:35,796 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:35,822 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774 2024-11-17T12:45:35,823 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:36,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741868_1045 (size=371515) 2024-11-17T12:45:36,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741868_1045 (size=371515) 2024-11-17T12:45:36,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741868_1045 (size=371515) 2024-11-17T12:45:36,060 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774, size=362.8 K (371515bytes) 2024-11-17T12:45:36,060 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774 2024-11-17T12:45:36,061 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774 after 0ms 2024-11-17T12:45:36,089 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:36,091 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774 took 31ms 2024-11-17T12:45:36,095 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731847535774.temp 2024-11-17T12:45:36,118 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774 so closing down 2024-11-17T12:45:36,118 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:36,134 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003001-wal.1731847535774.temp 2024-11-17T12:45:36,146 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:36,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741869_1046 (size=371515) 2024-11-17T12:45:36,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741869_1046 (size=371515) 2024-11-17T12:45:36,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741869_1046 (size=371515) 2024-11-17T12:45:36,151 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003001-wal.1731847535774.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-11-17T12:45:36,153 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003001-wal.1731847535774.temp to hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000006000 2024-11-17T12:45:36,153 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-17T12:45:36,153 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774, journal: Splitting hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774, size=362.8 K (371515bytes) at 1731847536060Split 1024 edits, skipped 0 edits. at 1731847536099 (+39 ms)Split 2048 edits, skipped 0 edits. at 1731847536107 (+8 ms)Finishing writing output for hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774 so closing down at 1731847536118 (+11 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003001-wal.1731847535774.temp at 1731847536134 (+16 ms)3 split writer threads finished at 1731847536146 (+12 ms)Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003001-wal.1731847535774.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1731847536151 (+5 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003001-wal.1731847535774.temp to hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000006000 at 1731847536153 (+2 ms)Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731847536153 2024-11-17T12:45:36,156 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847535774 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847535774 2024-11-17T12:45:36,156 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000006000 2024-11-17T12:45:36,157 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:36,158 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/test2727-manual,16010,1731847535256, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:36,171 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847536159, exclude list is [], retry=0 2024-11-17T12:45:36,174 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:36,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:36,175 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:36,203 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731847535256/wal.1731847536159 2024-11-17T12:45:36,204 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:45:36,204 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6bb0502de7f710f05d2f473083767b48, NAME => 'test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:36,204 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:36,204 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,204 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,206 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,207 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bb0502de7f710f05d2f473083767b48 columnFamilyName a 2024-11-17T12:45:36,207 DEBUG [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,208 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(327): Store=6bb0502de7f710f05d2f473083767b48/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,208 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,209 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bb0502de7f710f05d2f473083767b48 columnFamilyName b 2024-11-17T12:45:36,209 DEBUG [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,209 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(327): Store=6bb0502de7f710f05d2f473083767b48/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,209 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,210 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6bb0502de7f710f05d2f473083767b48 columnFamilyName c 2024-11-17T12:45:36,210 DEBUG [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,211 INFO [StoreOpener-6bb0502de7f710f05d2f473083767b48-1 {}] regionserver.HStore(327): Store=6bb0502de7f710f05d2f473083767b48/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,211 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,212 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,215 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003000 2024-11-17T12:45:36,240 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:36,266 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003000 2024-11-17T12:45:36,268 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000006000 2024-11-17T12:45:36,293 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:36,319 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000006000 2024-11-17T12:45:36,319 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6bb0502de7f710f05d2f473083767b48 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-17T12:45:36,337 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/a/cac0ee5a17764c40b437ea5a178ff7fd is 41, key is test2727/a:100/1731847535827/Put/seqid=0 2024-11-17T12:45:36,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741871_1048 (size=84227) 2024-11-17T12:45:36,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741871_1048 (size=84227) 2024-11-17T12:45:36,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741871_1048 (size=84227) 2024-11-17T12:45:36,345 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/a/cac0ee5a17764c40b437ea5a178ff7fd 2024-11-17T12:45:36,378 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/b/a0eec5de1cca4849b194ee90d2d1d03d is 41, key is test2727/b:100/1731847535875/Put/seqid=0 2024-11-17T12:45:36,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741872_1049 (size=84609) 2024-11-17T12:45:36,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741872_1049 (size=84609) 2024-11-17T12:45:36,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741872_1049 (size=84609) 2024-11-17T12:45:36,384 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/b/a0eec5de1cca4849b194ee90d2d1d03d 2024-11-17T12:45:36,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/c/65cb29bbcf9547cb8bb474a14e60807c is 41, key is test2727/c:100/1731847535943/Put/seqid=0 2024-11-17T12:45:36,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741873_1050 (size=84609) 2024-11-17T12:45:36,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741873_1050 (size=84609) 2024-11-17T12:45:36,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741873_1050 (size=84609) 2024-11-17T12:45:36,421 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/c/65cb29bbcf9547cb8bb474a14e60807c 2024-11-17T12:45:36,431 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/a/cac0ee5a17764c40b437ea5a178ff7fd as hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/a/cac0ee5a17764c40b437ea5a178ff7fd 2024-11-17T12:45:36,439 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/a/cac0ee5a17764c40b437ea5a178ff7fd, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-17T12:45:36,440 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/b/a0eec5de1cca4849b194ee90d2d1d03d as hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/b/a0eec5de1cca4849b194ee90d2d1d03d 2024-11-17T12:45:36,448 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/b/a0eec5de1cca4849b194ee90d2d1d03d, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-17T12:45:36,450 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/.tmp/c/65cb29bbcf9547cb8bb474a14e60807c as hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/c/65cb29bbcf9547cb8bb474a14e60807c 2024-11-17T12:45:36,458 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/c/65cb29bbcf9547cb8bb474a14e60807c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-17T12:45:36,458 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 6bb0502de7f710f05d2f473083767b48 in 139ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-17T12:45:36,459 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000003000 2024-11-17T12:45:36,460 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/0000000000000006000 2024-11-17T12:45:36,461 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,461 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,462 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:36,464 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6bb0502de7f710f05d2f473083767b48 2024-11-17T12:45:36,467 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/test2727/6bb0502de7f710f05d2f473083767b48/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-17T12:45:36,468 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6bb0502de7f710f05d2f473083767b48; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64671349, jitterRate=-0.03632180392742157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:36,469 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6bb0502de7f710f05d2f473083767b48: Writing region info on filesystem at 1731847536204Initializing all the Stores at 1731847536205 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536206 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536206Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536206Obtaining lock to block concurrent updates at 1731847536319 (+113 ms)Preparing flush snapshotting stores in 6bb0502de7f710f05d2f473083767b48 at 1731847536319Finished memstore snapshotting test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731847536319Flushing stores of test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. at 1731847536319Flushing 6bb0502de7f710f05d2f473083767b48/a: creating writer at 1731847536320 (+1 ms)Flushing 6bb0502de7f710f05d2f473083767b48/a: appending metadata at 1731847536337 (+17 ms)Flushing 6bb0502de7f710f05d2f473083767b48/a: closing flushed file at 1731847536337Flushing 6bb0502de7f710f05d2f473083767b48/b: creating writer at 1731847536352 (+15 ms)Flushing 6bb0502de7f710f05d2f473083767b48/b: appending metadata at 1731847536376 (+24 ms)Flushing 6bb0502de7f710f05d2f473083767b48/b: closing flushed file at 1731847536377 (+1 ms)Flushing 6bb0502de7f710f05d2f473083767b48/c: creating writer at 1731847536391 (+14 ms)Flushing 6bb0502de7f710f05d2f473083767b48/c: appending metadata at 1731847536413 (+22 ms)Flushing 6bb0502de7f710f05d2f473083767b48/c: closing flushed file at 1731847536413Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d435848: reopening flushed file at 1731847536429 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@492d482e: reopening flushed file at 1731847536439 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5793b199: reopening flushed file at 1731847536449 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 6bb0502de7f710f05d2f473083767b48 in 139ms, sequenceid=6000, compaction requested=false; wal=null at 1731847536458 (+9 ms)Cleaning up temporary data from old regions at 1731847536461 (+3 ms)Region opened successfully at 1731847536469 (+8 ms) 2024-11-17T12:45:36,470 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-17T12:45:36,470 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6bb0502de7f710f05d2f473083767b48, disabling compactions & flushes 2024-11-17T12:45:36,470 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:36,471 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:36,471 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. after waiting 0 ms 2024-11-17T12:45:36,471 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:36,472 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731847535258.6bb0502de7f710f05d2f473083767b48. 2024-11-17T12:45:36,472 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6bb0502de7f710f05d2f473083767b48: Waiting for close lock at 1731847536470Disabling compacts and flushes for region at 1731847536470Disabling writes for close at 1731847536471 (+1 ms)Writing region close event to WAL at 1731847536472 (+1 ms)Closed at 1731847536472 2024-11-17T12:45:36,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741870_1047 (size=175) 2024-11-17T12:45:36,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741870_1047 (size=175) 2024-11-17T12:45:36,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741870_1047 (size=175) 2024-11-17T12:45:36,477 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:36,478 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731847536159) 2024-11-17T12:45:36,490 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403 (was 399) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:40236 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:39032 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:50126 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=262 (was 262), ProcessCount=11 (was 11), AvailableMemoryMB=6145 (was 6361) 2024-11-17T12:45:36,501 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=262, ProcessCount=11, AvailableMemoryMB=6145 2024-11-17T12:45:36,515 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:36,521 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:36,522 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731847536522 2024-11-17T12:45:36,552 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522 2024-11-17T12:45:36,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:36,556 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0319f00959adf0beac288ba43882eb3, NAME => 'testSequentialEditLogSeqNum,,1731847536516.d0319f00959adf0beac288ba43882eb3.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:36,557 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731847536516.d0319f00959adf0beac288ba43882eb3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:36,557 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,557 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,558 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3 doesn't exist for region: d0319f00959adf0beac288ba43882eb3 on table testSequentialEditLogSeqNum 2024-11-17T12:45:36,558 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: d0319f00959adf0beac288ba43882eb3 on table testSequentialEditLogSeqNum 2024-11-17T12:45:36,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741875_1052 (size=62) 2024-11-17T12:45:36,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741875_1052 (size=62) 2024-11-17T12:45:36,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741875_1052 (size=62) 2024-11-17T12:45:36,572 INFO [StoreOpener-d0319f00959adf0beac288ba43882eb3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,573 INFO [StoreOpener-d0319f00959adf0beac288ba43882eb3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0319f00959adf0beac288ba43882eb3 columnFamilyName a 2024-11-17T12:45:36,573 DEBUG [StoreOpener-d0319f00959adf0beac288ba43882eb3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,574 INFO [StoreOpener-d0319f00959adf0beac288ba43882eb3-1 {}] regionserver.HStore(327): Store=d0319f00959adf0beac288ba43882eb3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,574 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,575 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,575 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,576 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,576 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,577 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0319f00959adf0beac288ba43882eb3 2024-11-17T12:45:36,580 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:36,580 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0319f00959adf0beac288ba43882eb3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61178213, jitterRate=-0.0883735865354538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:36,581 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0319f00959adf0beac288ba43882eb3: Writing region info on filesystem at 1731847536557Initializing all the Stores at 1731847536571 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536571Cleaning up temporary data from old regions at 1731847536576 (+5 ms)Region opened successfully at 1731847536581 (+5 ms) 2024-11-17T12:45:36,592 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0319f00959adf0beac288ba43882eb3 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-17T12:45:36,612 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/.tmp/a/8bcec9a1ff904ab9b26e26695765df48 is 81, key is testSequentialEditLogSeqNum/a:x0/1731847536581/Put/seqid=0 2024-11-17T12:45:36,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741876_1053 (size=5833) 2024-11-17T12:45:36,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741876_1053 (size=5833) 2024-11-17T12:45:36,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741876_1053 (size=5833) 2024-11-17T12:45:36,620 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/.tmp/a/8bcec9a1ff904ab9b26e26695765df48 2024-11-17T12:45:36,626 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/.tmp/a/8bcec9a1ff904ab9b26e26695765df48 as hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/a/8bcec9a1ff904ab9b26e26695765df48 2024-11-17T12:45:36,634 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/a/8bcec9a1ff904ab9b26e26695765df48, entries=10, sequenceid=13, filesize=5.7 K 2024-11-17T12:45:36,636 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for d0319f00959adf0beac288ba43882eb3 in 44ms, sequenceid=13, compaction requested=false 2024-11-17T12:45:36,636 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d0319f00959adf0beac288ba43882eb3: 2024-11-17T12:45:36,643 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T12:45:36,643 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T12:45:36,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T12:45:36,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T12:45:36,644 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-17T12:45:36,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741874_1051 (size=3256) 2024-11-17T12:45:36,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741874_1051 (size=3256) 2024-11-17T12:45:36,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741874_1051 (size=3256) 2024-11-17T12:45:36,669 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522, size=3.2 K (3256bytes) 2024-11-17T12:45:36,669 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522 2024-11-17T12:45:36,670 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522 after 1ms 2024-11-17T12:45:36,695 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:36,696 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522 took 26ms 2024-11-17T12:45:36,699 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522 so closing down 2024-11-17T12:45:36,699 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:36,700 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731847536522.temp 2024-11-17T12:45:36,726 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000003-wal.1731847536522.temp 2024-11-17T12:45:36,727 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:36,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741877_1054 (size=2650) 2024-11-17T12:45:36,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741877_1054 (size=2650) 2024-11-17T12:45:36,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741877_1054 (size=2650) 2024-11-17T12:45:36,737 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000003-wal.1731847536522.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:36,738 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000003-wal.1731847536522.temp to hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000020 2024-11-17T12:45:36,738 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-17T12:45:36,738 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522, journal: Splitting hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522, size=3.2 K (3256bytes) at 1731847536669Finishing writing output for hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522 so closing down at 1731847536699 (+30 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000003-wal.1731847536522.temp at 1731847536726 (+27 ms)3 split writer threads finished at 1731847536727 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000003-wal.1731847536522.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731847536737 (+10 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000003-wal.1731847536522.temp to hdfs://localhost:37973/hbase/data/default/testSequentialEditLogSeqNum/d0319f00959adf0beac288ba43882eb3/recovered.edits/0000000000000000020 at 1731847536738 (+1 ms)Processed 17 edits across 1 Regions in 42 ms; skipped=2; WAL=hdfs://localhost:37973/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731847536514/wal.1731847536522, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731847536738 2024-11-17T12:45:36,751 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:40236 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:50126 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=262 (was 262), ProcessCount=11 (was 11), AvailableMemoryMB=6142 (was 6145) 2024-11-17T12:45:36,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=262, ProcessCount=11, AvailableMemoryMB=6141 2024-11-17T12:45:36,777 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:36,779 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:36,815 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:36,818 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-88058767, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-88058767, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:36,833 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-88058767/hregion-88058767.1731847536819, exclude list is [], retry=0 2024-11-17T12:45:36,836 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:36,836 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:36,836 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:36,859 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-88058767/hregion-88058767.1731847536819 2024-11-17T12:45:36,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:36,860 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b9655d1b391fad979aa5bbb1868e01e7, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:36,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741879_1056 (size=70) 2024-11-17T12:45:36,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741879_1056 (size=70) 2024-11-17T12:45:36,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741879_1056 (size=70) 2024-11-17T12:45:36,870 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:36,872 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,873 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName a 2024-11-17T12:45:36,873 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,874 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,874 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,875 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName b 2024-11-17T12:45:36,875 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,876 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,876 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,877 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName c 2024-11-17T12:45:36,877 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,877 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,877 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,879 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,879 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,880 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:36,881 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,884 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:36,885 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9655d1b391fad979aa5bbb1868e01e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61993774, jitterRate=-0.07622078061103821}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:36,885 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9655d1b391fad979aa5bbb1868e01e7: Writing region info on filesystem at 1731847536870Initializing all the Stores at 1731847536871 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536871Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536872 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536872Cleaning up temporary data from old regions at 1731847536879 (+7 ms)Region opened successfully at 1731847536885 (+6 ms) 2024-11-17T12:45:36,886 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b9655d1b391fad979aa5bbb1868e01e7, disabling compactions & flushes 2024-11-17T12:45:36,886 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:36,886 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:36,886 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. after waiting 0 ms 2024-11-17T12:45:36,886 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:36,886 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:36,886 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b9655d1b391fad979aa5bbb1868e01e7: Waiting for close lock at 1731847536885Disabling compacts and flushes for region at 1731847536885Disabling writes for close at 1731847536886 (+1 ms)Writing region close event to WAL at 1731847536886Closed at 1731847536886 2024-11-17T12:45:36,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741878_1055 (size=175) 2024-11-17T12:45:36,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741878_1055 (size=175) 2024-11-17T12:45:36,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741878_1055 (size=175) 2024-11-17T12:45:36,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:36,894 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-88058767:(num 1731847536819) 2024-11-17T12:45:36,894 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:36,896 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:36,913 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897, exclude list is [], retry=0 2024-11-17T12:45:36,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:36,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:36,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:36,942 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 2024-11-17T12:45:36,942 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:36,942 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b9655d1b391fad979aa5bbb1868e01e7, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:36,943 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:36,943 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,943 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,945 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,946 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName a 2024-11-17T12:45:36,946 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,946 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,946 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,947 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName b 2024-11-17T12:45:36,947 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,948 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,948 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,949 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName c 2024-11-17T12:45:36,949 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:36,949 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:36,949 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,951 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,952 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,952 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,953 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:36,954 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:36,955 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9655d1b391fad979aa5bbb1868e01e7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74994193, jitterRate=0.11750055849552155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:36,955 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9655d1b391fad979aa5bbb1868e01e7: Writing region info on filesystem at 1731847536943Initializing all the Stores at 1731847536944 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536944Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536944Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847536944Cleaning up temporary data from old regions at 1731847536952 (+8 ms)Region opened successfully at 1731847536955 (+3 ms) 2024-11-17T12:45:36,960 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731847536959/Put/seqid=0 2024-11-17T12:45:36,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741881_1058 (size=4826) 2024-11-17T12:45:36,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741881_1058 (size=4826) 2024-11-17T12:45:36,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741881_1058 (size=4826) 2024-11-17T12:45:36,974 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37973/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in b9655d1b391fad979aa5bbb1868e01e7/a 2024-11-17T12:45:36,982 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-17T12:45:36,982 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-17T12:45:36,982 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b9655d1b391fad979aa5bbb1868e01e7: 2024-11-17T12:45:36,984 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/a/79cf954b980849ca850b7f16dda1bb19_SeqId_3_ 2024-11-17T12:45:36,985 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37973/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b9655d1b391fad979aa5bbb1868e01e7/a as hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/a/79cf954b980849ca850b7f16dda1bb19_SeqId_3_ - updating store file list. 2024-11-17T12:45:36,990 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 79cf954b980849ca850b7f16dda1bb19_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-17T12:45:36,991 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/a/79cf954b980849ca850b7f16dda1bb19_SeqId_3_ into b9655d1b391fad979aa5bbb1868e01e7/a 2024-11-17T12:45:36,991 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37973/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b9655d1b391fad979aa5bbb1868e01e7/a (new location: hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/a/79cf954b980849ca850b7f16dda1bb19_SeqId_3_) 2024-11-17T12:45:37,027 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897, size=0 (0bytes) 2024-11-17T12:45:37,027 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 might be still open, length is 0 2024-11-17T12:45:37,027 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 2024-11-17T12:45:37,028 WARN [IPC Server handler 0 on default port 37973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-17T12:45:37,028 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 after 1ms 2024-11-17T12:45:39,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:39146 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34421:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39146 dst: /127.0.0.1:34421 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34421 remote=/127.0.0.1:39146]. Total timeout mills is 60000, 57722 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:39,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:50200 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:44483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50200 dst: /127.0.0.1:44483 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:39,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:40440 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42393:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40440 dst: /127.0.0.1:42393 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:39,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741880_1059 (size=606) 2024-11-17T12:45:39,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741880_1059 (size=606) 2024-11-17T12:45:39,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741880_1059 (size=606) 2024-11-17T12:45:41,029 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 after 4002ms 2024-11-17T12:45:41,060 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:41,060 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 took 4033ms 2024-11-17T12:45:41,063 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897; continuing. 2024-11-17T12:45:41,063 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 so closing down 2024-11-17T12:45:41,063 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:41,065 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731847536897.temp 2024-11-17T12:45:41,088 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005-wal.1731847536897.temp 2024-11-17T12:45:41,088 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741882_1060 (size=329) 2024-11-17T12:45:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741882_1060 (size=329) 2024-11-17T12:45:41,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741882_1060 (size=329) 2024-11-17T12:45:41,095 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005-wal.1731847536897.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:41,096 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005-wal.1731847536897.temp to hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005 2024-11-17T12:45:41,096 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897, size=0, length=0, corrupted=false, cancelled=false 2024-11-17T12:45:41,097 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897, journal: Splitting hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897, size=0 (0bytes) at 1731847537027Finishing writing output for hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 so closing down at 1731847541063 (+4036 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005-wal.1731847536897.temp at 1731847541088 (+25 ms)3 split writer threads finished at 1731847541088Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005-wal.1731847536897.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731847541095 (+7 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005-wal.1731847536897.temp to hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005 at 1731847541096 (+1 ms)Processed 2 edits across 1 Regions in 35 ms; skipped=1; WAL=hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897, size=0, length=0, corrupted=false, cancelled=false at 1731847541097 (+1 ms) 2024-11-17T12:45:41,098 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847536897 2024-11-17T12:45:41,099 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005 2024-11-17T12:45:41,099 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:41,101 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:41,113 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847541102, exclude list is [], retry=0 2024-11-17T12:45:41,116 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:41,116 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:41,117 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:41,139 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847541102 2024-11-17T12:45:41,140 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:41,140 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b9655d1b391fad979aa5bbb1868e01e7, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:41,140 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:41,140 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,140 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,141 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,142 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName a 2024-11-17T12:45:41,142 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,148 DEBUG [StoreFileOpener-b9655d1b391fad979aa5bbb1868e01e7-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 79cf954b980849ca850b7f16dda1bb19_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-17T12:45:41,149 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/a/79cf954b980849ca850b7f16dda1bb19_SeqId_3_ 2024-11-17T12:45:41,149 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:41,149 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,150 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName b 2024-11-17T12:45:41,150 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,151 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:41,151 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,152 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9655d1b391fad979aa5bbb1868e01e7 columnFamilyName c 2024-11-17T12:45:41,152 DEBUG [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,152 INFO [StoreOpener-b9655d1b391fad979aa5bbb1868e01e7-1 {}] regionserver.HStore(327): Store=b9655d1b391fad979aa5bbb1868e01e7/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:41,153 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,153 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,155 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,155 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005 2024-11-17T12:45:41,182 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:41,183 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005 2024-11-17T12:45:41,183 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b9655d1b391fad979aa5bbb1868e01e7 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-17T12:45:41,196 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/.tmp/a/ff69880b5e0a417cb87f15f8d722048d is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731847536995/Put/seqid=0 2024-11-17T12:45:41,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741884_1062 (size=5149) 2024-11-17T12:45:41,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741884_1062 (size=5149) 2024-11-17T12:45:41,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741884_1062 (size=5149) 2024-11-17T12:45:41,203 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/.tmp/a/ff69880b5e0a417cb87f15f8d722048d 2024-11-17T12:45:41,210 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/.tmp/a/ff69880b5e0a417cb87f15f8d722048d as hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/a/ff69880b5e0a417cb87f15f8d722048d 2024-11-17T12:45:41,216 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/a/ff69880b5e0a417cb87f15f8d722048d, entries=1, sequenceid=5, filesize=5.0 K 2024-11-17T12:45:41,217 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b9655d1b391fad979aa5bbb1868e01e7 in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-11-17T12:45:41,217 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/0000000000000000005 2024-11-17T12:45:41,219 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,219 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,219 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:41,221 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9655d1b391fad979aa5bbb1868e01e7 2024-11-17T12:45:41,223 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9655d1b391fad979aa5bbb1868e01e7/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-17T12:45:41,224 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9655d1b391fad979aa5bbb1868e01e7; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73556763, jitterRate=0.09608118236064911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:41,225 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9655d1b391fad979aa5bbb1868e01e7: Writing region info on filesystem at 1731847541140Initializing all the Stores at 1731847541141 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847541141Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847541141Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847541141Obtaining lock to block concurrent updates at 1731847541183 (+42 ms)Preparing flush snapshotting stores in b9655d1b391fad979aa5bbb1868e01e7 at 1731847541183Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731847541183Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. at 1731847541183Flushing b9655d1b391fad979aa5bbb1868e01e7/a: creating writer at 1731847541183Flushing b9655d1b391fad979aa5bbb1868e01e7/a: appending metadata at 1731847541196 (+13 ms)Flushing b9655d1b391fad979aa5bbb1868e01e7/a: closing flushed file at 1731847541196Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cddfd21: reopening flushed file at 1731847541209 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b9655d1b391fad979aa5bbb1868e01e7 in 34ms, sequenceid=5, compaction requested=false; wal=null at 1731847541217 (+8 ms)Cleaning up temporary data from old regions at 1731847541219 (+2 ms)Region opened successfully at 1731847541225 (+6 ms) 2024-11-17T12:45:41,228 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b9655d1b391fad979aa5bbb1868e01e7, disabling compactions & flushes 2024-11-17T12:45:41,228 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:41,228 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:41,228 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. after waiting 0 ms 2024-11-17T12:45:41,228 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:41,229 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731847536778.b9655d1b391fad979aa5bbb1868e01e7. 2024-11-17T12:45:41,229 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b9655d1b391fad979aa5bbb1868e01e7: Waiting for close lock at 1731847541228Disabling compacts and flushes for region at 1731847541228Disabling writes for close at 1731847541228Writing region close event to WAL at 1731847541229 (+1 ms)Closed at 1731847541229 2024-11-17T12:45:41,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741883_1061 (size=175) 2024-11-17T12:45:41,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741883_1061 (size=175) 2024-11-17T12:45:41,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741883_1061 (size=175) 2024-11-17T12:45:41,234 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:41,234 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731847541102) 2024-11-17T12:45:41,245 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409 (was 408) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:37973 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_610386728_22 at /127.0.0.1:39170 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_610386728_22 at /127.0.0.1:50230 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:37973 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=241 (was 262), ProcessCount=11 (was 11), AvailableMemoryMB=6138 (was 6141) 2024-11-17T12:45:41,256 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=241, ProcessCount=11, AvailableMemoryMB=6137 2024-11-17T12:45:41,272 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:41,276 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T12:45:41,280 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0d751a622496,38549,1731847525223 2024-11-17T12:45:41,282 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1cf5e11e 2024-11-17T12:45:41,283 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T12:45:41,285 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T12:45:41,288 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T12:45:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-17T12:45:41,295 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T12:45:41,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-17T12:45:41,297 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,298 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T12:45:41,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T12:45:41,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741885_1063 (size=694) 2024-11-17T12:45:41,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741885_1063 (size=694) 2024-11-17T12:45:41,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741885_1063 (size=694) 2024-11-17T12:45:41,310 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8e8bd106579494c8f12507d73483b546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8 2024-11-17T12:45:41,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741886_1064 (size=77) 2024-11-17T12:45:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741886_1064 (size=77) 2024-11-17T12:45:41,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741886_1064 (size=77) 2024-11-17T12:45:41,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:41,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 8e8bd106579494c8f12507d73483b546, disabling compactions & flushes 2024-11-17T12:45:41,319 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. after waiting 0 ms 2024-11-17T12:45:41,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,319 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,319 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8e8bd106579494c8f12507d73483b546: Waiting for close lock at 1731847541319Disabling compacts and flushes for region at 1731847541319Disabling writes for close at 1731847541319Writing region close event to WAL at 1731847541319Closed at 1731847541319 2024-11-17T12:45:41,321 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T12:45:41,326 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731847541321"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731847541321"}]},"ts":"1731847541321"} 2024-11-17T12:45:41,330 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T12:45:41,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T12:45:41,333 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847541331"}]},"ts":"1731847541331"} 2024-11-17T12:45:41,337 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-17T12:45:41,337 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0d751a622496=0} racks are {/default-rack=0} 2024-11-17T12:45:41,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T12:45:41,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T12:45:41,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-17T12:45:41,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T12:45:41,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T12:45:41,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-17T12:45:41,339 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T12:45:41,339 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T12:45:41,339 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-17T12:45:41,339 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T12:45:41,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN}] 2024-11-17T12:45:41,343 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN 2024-11-17T12:45:41,344 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN; state=OFFLINE, location=0d751a622496,44359,1731847526081; forceNewPlan=false, retain=false 2024-11-17T12:45:41,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T12:45:41,496 INFO [0d751a622496:38549 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T12:45:41,497 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPENING, regionLocation=0d751a622496,44359,1731847526081 2024-11-17T12:45:41,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN because future has completed 2024-11-17T12:45:41,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081}] 2024-11-17T12:45:41,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T12:45:41,666 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,666 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e8bd106579494c8f12507d73483b546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:41,667 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,667 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:41,668 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,668 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,670 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,672 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf1 2024-11-17T12:45:41,673 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,673 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:41,674 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,676 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf2 2024-11-17T12:45:41,676 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,677 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:41,677 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,678 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,678 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,679 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,679 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,680 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-17T12:45:41,681 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,684 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:41,684 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8e8bd106579494c8f12507d73483b546; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59197736, jitterRate=-0.11788499355316162}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-17T12:45:41,684 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,685 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8e8bd106579494c8f12507d73483b546: Running coprocessor pre-open hook at 1731847541668Writing region info on filesystem at 1731847541668Initializing all the Stores at 1731847541669 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847541669Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847541670 (+1 ms)Cleaning up temporary data from old regions at 1731847541679 (+9 ms)Running coprocessor post-open hooks at 1731847541684 (+5 ms)Region opened successfully at 1731847541685 (+1 ms) 2024-11-17T12:45:41,687 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., pid=6, masterSystemTime=1731847541654 2024-11-17T12:45:41,689 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,689 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,690 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPEN, openSeqNum=2, regionLocation=0d751a622496,44359,1731847526081 2024-11-17T12:45:41,693 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081 because future has completed 2024-11-17T12:45:41,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T12:45:41,698 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081 in 194 msec 2024-11-17T12:45:41,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T12:45:41,702 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN in 357 msec 2024-11-17T12:45:41,703 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T12:45:41,703 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731847541703"}]},"ts":"1731847541703"} 2024-11-17T12:45:41,706 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-17T12:45:41,707 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T12:45:41,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 418 msec 2024-11-17T12:45:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T12:45:41,931 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-17T12:45:41,931 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-17T12:45:41,932 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T12:45:41,938 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-17T12:45:41,939 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T12:45:41,939 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-17T12:45:41,952 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=2] 2024-11-17T12:45:41,968 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8e8bd106579494c8f12507d73483b546, source=0d751a622496,44359,1731847526081, destination=0d751a622496,38707,1731847525964, warming up region on 0d751a622496,38707,1731847525964 2024-11-17T12:45:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T12:45:41,972 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8e8bd106579494c8f12507d73483b546, source=0d751a622496,44359,1731847526081, destination=0d751a622496,38707,1731847525964, running balancer 2024-11-17T12:45:41,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54681, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T12:45:41,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE 2024-11-17T12:45:41,974 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE 2024-11-17T12:45:41,976 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=CLOSING, regionLocation=0d751a622496,44359,1731847526081 2024-11-17T12:45:41,979 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8e8bd106579494c8f12507d73483b546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:41,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE because future has completed 2024-11-17T12:45:41,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:41,980 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,980 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T12:45:41,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081}] 2024-11-17T12:45:41,981 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf1 2024-11-17T12:45:41,981 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,982 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:41,982 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:41,983 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf2 2024-11-17T12:45:41,983 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:41,984 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(1722): Closing 8e8bd106579494c8f12507d73483b546, disabling compactions & flushes 2024-11-17T12:45:41,984 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. after waiting 0 ms 2024-11-17T12:45:41,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,985 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:41,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] regionserver.HRegion(1676): Region close journal for 8e8bd106579494c8f12507d73483b546: Waiting for close lock at 1731847541984Disabling compacts and flushes for region at 1731847541984Disabling writes for close at 1731847541984Writing region close event to WAL at 1731847541985 (+1 ms)Closed at 1731847541985 2024-11-17T12:45:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-17T12:45:42,139 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,140 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T12:45:42,140 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8e8bd106579494c8f12507d73483b546, disabling compactions & flushes 2024-11-17T12:45:42,141 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,141 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,141 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. after waiting 0 ms 2024-11-17T12:45:42,141 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,141 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 8e8bd106579494c8f12507d73483b546 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-17T12:45:42,160 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf1/ff6d3afac1224b2d89a36ad58c349d6c is 35, key is r1/cf1:q/1731847541954/Put/seqid=0 2024-11-17T12:45:42,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741887_1065 (size=4783) 2024-11-17T12:45:42,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741887_1065 (size=4783) 2024-11-17T12:45:42,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741887_1065 (size=4783) 2024-11-17T12:45:42,168 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf1/ff6d3afac1224b2d89a36ad58c349d6c 2024-11-17T12:45:42,177 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf1/ff6d3afac1224b2d89a36ad58c349d6c as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c 2024-11-17T12:45:42,184 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c, entries=1, sequenceid=5, filesize=4.7 K 2024-11-17T12:45:42,185 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 8e8bd106579494c8f12507d73483b546 in 44ms, sequenceid=5, compaction requested=false 2024-11-17T12:45:42,186 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-17T12:45:42,191 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-17T12:45:42,193 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,193 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8e8bd106579494c8f12507d73483b546: Waiting for close lock at 1731847542140Running coprocessor pre-close hooks at 1731847542140Disabling compacts and flushes for region at 1731847542140Disabling writes for close at 1731847542141 (+1 ms)Obtaining lock to block concurrent updates at 1731847542141Preparing flush snapshotting stores in 8e8bd106579494c8f12507d73483b546 at 1731847542141Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731847542142 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. at 1731847542142Flushing 8e8bd106579494c8f12507d73483b546/cf1: creating writer at 1731847542142Flushing 8e8bd106579494c8f12507d73483b546/cf1: appending metadata at 1731847542160 (+18 ms)Flushing 8e8bd106579494c8f12507d73483b546/cf1: closing flushed file at 1731847542160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1c3959: reopening flushed file at 1731847542176 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 8e8bd106579494c8f12507d73483b546 in 44ms, sequenceid=5, compaction requested=false at 1731847542185 (+9 ms)Writing region close event to WAL at 1731847542187 (+2 ms)Running coprocessor post-close hooks at 1731847542191 (+4 ms)Closed at 1731847542193 (+2 ms) 2024-11-17T12:45:42,194 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8e8bd106579494c8f12507d73483b546 move to 0d751a622496,38707,1731847525964 record at close sequenceid=5 2024-11-17T12:45:42,198 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,198 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=CLOSED 2024-11-17T12:45:42,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081 because future has completed 2024-11-17T12:45:42,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T12:45:42,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081 in 223 msec 2024-11-17T12:45:42,208 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE; state=CLOSED, location=0d751a622496,38707,1731847525964; forceNewPlan=false, retain=false 2024-11-17T12:45:42,358 INFO [0d751a622496:38549 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T12:45:42,359 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPENING, regionLocation=0d751a622496,38707,1731847525964 2024-11-17T12:45:42,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE because future has completed 2024-11-17T12:45:42,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,38707,1731847525964}] 2024-11-17T12:45:42,528 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,528 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e8bd106579494c8f12507d73483b546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:42,529 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,529 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:42,530 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,530 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,532 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,533 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf1 2024-11-17T12:45:42,534 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:42,543 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c 2024-11-17T12:45:42,544 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:42,544 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,545 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf2 2024-11-17T12:45:42,545 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:42,545 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:42,546 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,547 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,548 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,549 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,549 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,550 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-17T12:45:42,551 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,552 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8e8bd106579494c8f12507d73483b546; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66712337, jitterRate=-0.00590871274471283}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-17T12:45:42,552 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,553 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8e8bd106579494c8f12507d73483b546: Running coprocessor pre-open hook at 1731847542530Writing region info on filesystem at 1731847542530Initializing all the Stores at 1731847542532 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847542532Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847542532Cleaning up temporary data from old regions at 1731847542549 (+17 ms)Running coprocessor post-open hooks at 1731847542552 (+3 ms)Region opened successfully at 1731847542553 (+1 ms) 2024-11-17T12:45:42,554 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., pid=9, masterSystemTime=1731847542520 2024-11-17T12:45:42,556 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,556 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,557 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPEN, openSeqNum=9, regionLocation=0d751a622496,38707,1731847525964 2024-11-17T12:45:42,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,38707,1731847525964 because future has completed 2024-11-17T12:45:42,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-17T12:45:42,564 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,38707,1731847525964 in 197 msec 2024-11-17T12:45:42,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE in 592 msec 2024-11-17T12:45:42,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T12:45:42,583 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T12:45:42,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:34660 deadline: 1731847602587, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=38707 startCode=1731847525964. As of locationSeqNum=5. 2024-11-17T12:45:42,596 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=38707 startCode=1731847525964. As of locationSeqNum=5. 2024-11-17T12:45:42,596 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=38707 startCode=1731847525964. As of locationSeqNum=5. 2024-11-17T12:45:42,596 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,38707,1731847525964, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=38707 startCode=1731847525964. As of locationSeqNum=5. 2024-11-17T12:45:42,710 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T12:45:42,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54886, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T12:45:42,720 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8e8bd106579494c8f12507d73483b546 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-17T12:45:42,736 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf1/8061fbcc2e3a4d0a850dbf2228a86054 is 29, key is r1/cf1:/1731847542713/DeleteFamily/seqid=0 2024-11-17T12:45:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741888_1066 (size=4906) 2024-11-17T12:45:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741888_1066 (size=4906) 2024-11-17T12:45:42,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741888_1066 (size=4906) 2024-11-17T12:45:42,745 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf1/8061fbcc2e3a4d0a850dbf2228a86054 2024-11-17T12:45:42,751 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8061fbcc2e3a4d0a850dbf2228a86054 2024-11-17T12:45:42,765 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf2/b937997db8c545e290360acc3434928c is 29, key is r1/cf2:/1731847542713/DeleteFamily/seqid=0 2024-11-17T12:45:42,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741889_1067 (size=4906) 2024-11-17T12:45:42,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741889_1067 (size=4906) 2024-11-17T12:45:42,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741889_1067 (size=4906) 2024-11-17T12:45:42,774 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf2/b937997db8c545e290360acc3434928c 2024-11-17T12:45:42,779 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b937997db8c545e290360acc3434928c 2024-11-17T12:45:42,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf1/8061fbcc2e3a4d0a850dbf2228a86054 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/8061fbcc2e3a4d0a850dbf2228a86054 2024-11-17T12:45:42,787 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8061fbcc2e3a4d0a850dbf2228a86054 2024-11-17T12:45:42,788 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/8061fbcc2e3a4d0a850dbf2228a86054, entries=1, sequenceid=12, filesize=4.8 K 2024-11-17T12:45:42,789 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf2/b937997db8c545e290360acc3434928c as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/b937997db8c545e290360acc3434928c 2024-11-17T12:45:42,795 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b937997db8c545e290360acc3434928c 2024-11-17T12:45:42,795 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/b937997db8c545e290360acc3434928c, entries=1, sequenceid=12, filesize=4.8 K 2024-11-17T12:45:42,797 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 8e8bd106579494c8f12507d73483b546 in 76ms, sequenceid=12, compaction requested=false 2024-11-17T12:45:42,797 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8e8bd106579494c8f12507d73483b546: 2024-11-17T12:45:42,799 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-17T12:45:42,800 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8e8bd106579494c8f12507d73483b546/cf1 is initiating major compaction (all files) 2024-11-17T12:45:42,801 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T12:45:42,801 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:42,801 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8e8bd106579494c8f12507d73483b546/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,802 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c, hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/8061fbcc2e3a4d0a850dbf2228a86054] into tmpdir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp, totalSize=9.5 K 2024-11-17T12:45:42,803 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ff6d3afac1224b2d89a36ad58c349d6c, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731847541954 2024-11-17T12:45:42,803 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8061fbcc2e3a4d0a850dbf2228a86054, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-17T12:45:42,817 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 8e8bd106579494c8f12507d73483b546#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T12:45:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741890_1068 (size=4626) 2024-11-17T12:45:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741890_1068 (size=4626) 2024-11-17T12:45:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741890_1068 (size=4626) 2024-11-17T12:45:42,832 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf1/0108370a997c447a840348e1a160d410 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/0108370a997c447a840348e1a160d410 2024-11-17T12:45:42,849 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 8e8bd106579494c8f12507d73483b546/cf1 of 8e8bd106579494c8f12507d73483b546 into 0108370a997c447a840348e1a160d410(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T12:45:42,849 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8e8bd106579494c8f12507d73483b546: 2024-11-17T12:45:42,850 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-17T12:45:42,850 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8e8bd106579494c8f12507d73483b546/cf2 is initiating major compaction (all files) 2024-11-17T12:45:42,850 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T12:45:42,850 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T12:45:42,850 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8e8bd106579494c8f12507d73483b546/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,850 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/b937997db8c545e290360acc3434928c] into tmpdir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp, totalSize=4.8 K 2024-11-17T12:45:42,851 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b937997db8c545e290360acc3434928c, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-17T12:45:42,858 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 8e8bd106579494c8f12507d73483b546#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T12:45:42,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741891_1069 (size=4592) 2024-11-17T12:45:42,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741891_1069 (size=4592) 2024-11-17T12:45:42,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741891_1069 (size=4592) 2024-11-17T12:45:42,875 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/.tmp/cf2/37e88620f6444cc198b9979272b7c2b5 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/37e88620f6444cc198b9979272b7c2b5 2024-11-17T12:45:42,884 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 8e8bd106579494c8f12507d73483b546/cf2 of 8e8bd106579494c8f12507d73483b546 into 37e88620f6444cc198b9979272b7c2b5(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T12:45:42,884 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8e8bd106579494c8f12507d73483b546: 2024-11-17T12:45:42,890 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8e8bd106579494c8f12507d73483b546, source=0d751a622496,38707,1731847525964, destination=0d751a622496,44359,1731847526081, warming up region on 0d751a622496,44359,1731847526081 2024-11-17T12:45:42,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8e8bd106579494c8f12507d73483b546, source=0d751a622496,38707,1731847525964, destination=0d751a622496,44359,1731847526081, running balancer 2024-11-17T12:45:42,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE 2024-11-17T12:45:42,892 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE 2024-11-17T12:45:42,894 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=CLOSING, regionLocation=0d751a622496,38707,1731847525964 2024-11-17T12:45:42,894 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8e8bd106579494c8f12507d73483b546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:42,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:42,896 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,896 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38549 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=0d751a622496,38707,1731847525964, table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-17T12:45:42,897 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf1 2024-11-17T12:45:42,897 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:42,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE because future has completed 2024-11-17T12:45:42,899 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T12:45:42,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,38707,1731847525964}] 2024-11-17T12:45:42,906 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/0108370a997c447a840348e1a160d410 2024-11-17T12:45:42,912 INFO [StoreFileOpener-8e8bd106579494c8f12507d73483b546-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8061fbcc2e3a4d0a850dbf2228a86054 2024-11-17T12:45:42,912 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/8061fbcc2e3a4d0a850dbf2228a86054 2024-11-17T12:45:42,919 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c 2024-11-17T12:45:42,920 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:42,920 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:42,921 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf2 2024-11-17T12:45:42,921 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:42,928 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/37e88620f6444cc198b9979272b7c2b5 2024-11-17T12:45:42,934 INFO [StoreFileOpener-8e8bd106579494c8f12507d73483b546-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b937997db8c545e290360acc3434928c 2024-11-17T12:45:42,934 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/b937997db8c545e290360acc3434928c 2024-11-17T12:45:42,934 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(1722): Closing 8e8bd106579494c8f12507d73483b546, disabling compactions & flushes 2024-11-17T12:45:42,935 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. after waiting 0 ms 2024-11-17T12:45:42,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:42,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44359 {}] regionserver.HRegion(1676): Region close journal for 8e8bd106579494c8f12507d73483b546: Waiting for close lock at 1731847542935Disabling compacts and flushes for region at 1731847542935Disabling writes for close at 1731847542935Writing region close event to WAL at 1731847542936 (+1 ms)Closed at 1731847542936 2024-11-17T12:45:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-17T12:45:43,054 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,054 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T12:45:43,054 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 8e8bd106579494c8f12507d73483b546, disabling compactions & flushes 2024-11-17T12:45:43,054 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,054 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,054 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. after waiting 0 ms 2024-11-17T12:45:43,054 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,055 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c, hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/8061fbcc2e3a4d0a850dbf2228a86054] to archive 2024-11-17T12:45:43,057 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T12:45:43,061 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/ff6d3afac1224b2d89a36ad58c349d6c 2024-11-17T12:45:43,063 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/8061fbcc2e3a4d0a850dbf2228a86054 to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/8061fbcc2e3a4d0a850dbf2228a86054 2024-11-17T12:45:43,074 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/b937997db8c545e290360acc3434928c] to archive 2024-11-17T12:45:43,076 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T12:45:43,078 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/b937997db8c545e290360acc3434928c to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/b937997db8c545e290360acc3434928c 2024-11-17T12:45:43,083 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-17T12:45:43,084 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,084 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 8e8bd106579494c8f12507d73483b546: Waiting for close lock at 1731847543054Running coprocessor pre-close hooks at 1731847543054Disabling compacts and flushes for region at 1731847543054Disabling writes for close at 1731847543054Writing region close event to WAL at 1731847543079 (+25 ms)Running coprocessor post-close hooks at 1731847543084 (+5 ms)Closed at 1731847543084 2024-11-17T12:45:43,084 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 8e8bd106579494c8f12507d73483b546 move to 0d751a622496,44359,1731847526081 record at close sequenceid=12 2024-11-17T12:45:43,087 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,088 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=CLOSED 2024-11-17T12:45:43,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,38707,1731847525964 because future has completed 2024-11-17T12:45:43,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-17T12:45:43,095 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,38707,1731847525964 in 193 msec 2024-11-17T12:45:43,096 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE; state=CLOSED, location=0d751a622496,44359,1731847526081; forceNewPlan=false, retain=false 2024-11-17T12:45:43,246 INFO [0d751a622496:38549 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T12:45:43,246 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPENING, regionLocation=0d751a622496,44359,1731847526081 2024-11-17T12:45:43,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE because future has completed 2024-11-17T12:45:43,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081}] 2024-11-17T12:45:43,406 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,407 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e8bd106579494c8f12507d73483b546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:43,407 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,407 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:43,407 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,407 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,409 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,410 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf1 2024-11-17T12:45:43,410 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:43,416 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/0108370a997c447a840348e1a160d410 2024-11-17T12:45:43,416 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:43,416 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,417 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf2 2024-11-17T12:45:43,417 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:43,423 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/37e88620f6444cc198b9979272b7c2b5 2024-11-17T12:45:43,423 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:43,424 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,425 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,426 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,427 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,428 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,428 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-17T12:45:43,430 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,431 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 8e8bd106579494c8f12507d73483b546; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60247817, jitterRate=-0.10223756730556488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-17T12:45:43,431 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,432 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 8e8bd106579494c8f12507d73483b546: Running coprocessor pre-open hook at 1731847543407Writing region info on filesystem at 1731847543407Initializing all the Stores at 1731847543408 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847543409 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847543409Cleaning up temporary data from old regions at 1731847543428 (+19 ms)Running coprocessor post-open hooks at 1731847543432 (+4 ms)Region opened successfully at 1731847543432 2024-11-17T12:45:43,433 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., pid=12, masterSystemTime=1731847543402 2024-11-17T12:45:43,435 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,435 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,436 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPEN, openSeqNum=18, regionLocation=0d751a622496,44359,1731847526081 2024-11-17T12:45:43,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081 because future has completed 2024-11-17T12:45:43,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-17T12:45:43,443 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,44359,1731847526081 in 190 msec 2024-11-17T12:45:43,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, REOPEN/MOVE in 552 msec 2024-11-17T12:45:43,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T12:45:43,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56882, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T12:45:43,499 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 0d751a622496,44359,1731847526081: testing ***** 2024-11-17T12:45:43,499 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-17T12:45:43,502 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-17T12:45:43,504 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-17T12:45:43,508 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-17T12:45:43,510 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-17T12:45:43,518 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 500821792 }, "NonHeapMemoryUsage": { "committed": 171769856, "init": 7667712, "max": -1, "used": 169203288 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "0d751a622496", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 4, "TotalCallTime_75th_percentile": 6, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "0d751a622496", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:59814", "tag.serverName": "0d751a622496,44359,1731847526081", "tag.clusterId": "e64d9651-704a-4260-becd-b88a4609dc90", "tag.Context": "regionserver", "tag.Hostname": "0d751a622496", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731847526081, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 3, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 2, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 1, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 2, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-17T12:45:43,521 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38549 {}] master.MasterRpcServices(700): 0d751a622496,44359,1731847526081 reported a fatal error: ***** ABORTING region server 0d751a622496,44359,1731847526081: testing ***** 2024-11-17T12:45:43,523 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,44359,1731847526081' ***** 2024-11-17T12:45:43,523 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-17T12:45:43,523 INFO [RS:1;0d751a622496:44359 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T12:45:43,524 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T12:45:43,524 INFO [RS:1;0d751a622496:44359 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-17T12:45:43,524 INFO [RS:1;0d751a622496:44359 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-17T12:45:43,524 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,524 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(956): aborting server 0d751a622496,44359,1731847526081 2024-11-17T12:45:43,524 INFO [RS:1;0d751a622496:44359 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T12:45:43,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38707 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:54886 deadline: 1731847603524, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=44359 startCode=1731847526081. As of locationSeqNum=12. 2024-11-17T12:45:43,525 INFO [RS:1;0d751a622496:44359 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0d751a622496:44359. 2024-11-17T12:45:43,525 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e8bd106579494c8f12507d73483b546, disabling compactions & flushes 2024-11-17T12:45:43,525 DEBUG [RS:1;0d751a622496:44359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T12:45:43,525 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,525 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,525 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. after waiting 0 ms 2024-11-17T12:45:43,525 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,525 DEBUG [RS:1;0d751a622496:44359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:45:43,525 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,38707,1731847525964, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,38707,1731847525964, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=44359 startCode=1731847526081. As of locationSeqNum=12. 2024-11-17T12:45:43,525 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,38707,1731847525964, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=44359 startCode=1731847526081. As of locationSeqNum=12. 2024-11-17T12:45:43,525 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,38707,1731847525964, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=0d751a622496 port=44359 startCode=1731847526081. As of locationSeqNum=12. 2024-11-17T12:45:43,526 INFO [RS:1;0d751a622496:44359 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T12:45:43,526 INFO [RS:1;0d751a622496:44359 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T12:45:43,526 INFO [RS:1;0d751a622496:44359 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T12:45:43,526 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T12:45:43,526 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-17T12:45:43,527 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T12:45:43,527 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1325): Online Regions={8e8bd106579494c8f12507d73483b546=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., 1588230740=hbase:meta,,1.1588230740} 2024-11-17T12:45:43,527 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T12:45:43,527 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T12:45:43,527 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T12:45:43,527 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T12:45:43,527 DEBUG [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:43,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0d751a622496,44359,1731847526081 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:43,527 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,527 ERROR [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-17T12:45:43,527 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e8bd106579494c8f12507d73483b546: Waiting for close lock at 1731847543524Running coprocessor pre-close hooks at 1731847543525 (+1 ms)Disabling compacts and flushes for region at 1731847543525Disabling writes for close at 1731847543525Writing region close event to WAL at 1731847543527 (+2 ms)Running coprocessor post-close hooks at 1731847543527Closed at 1731847543527 2024-11-17T12:45:43,527 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T12:45:43,527 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T12:45:43,527 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:43,528 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731847543526Running coprocessor pre-close hooks at 1731847543526Disabling compacts and flushes for region at 1731847543526Disabling writes for close at 1731847543527 (+1 ms)Writing region close event to WAL at 1731847543527Running coprocessor post-close hooks at 1731847543527Closed at 1731847543527 2024-11-17T12:45:43,528 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T12:45:43,528 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0d751a622496,44359,1731847526081 aborting 2024-11-17T12:45:43,528 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0d751a622496,44359,1731847526081 aborting 2024-11-17T12:45:43,528 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 from cache 2024-11-17T12:45:43,539 INFO [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T12:45:43,539 INFO [regionserver/0d751a622496:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T12:45:43,552 INFO [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T12:45:43,639 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:43,641 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1] 2024-11-17T12:45:43,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0d751a622496,44359,1731847526081 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:43,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0d751a622496,44359,1731847526081 aborting 2024-11-17T12:45:43,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 0d751a622496,44359,1731847526081 aborting 2024-11-17T12:45:43,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 from cache 2024-11-17T12:45:43,727 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,44359,1731847526081; all regions closed. 2024-11-17T12:45:43,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741836_1012 (size=7323) 2024-11-17T12:45:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741836_1012 (size=7323) 2024-11-17T12:45:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741836_1012 (size=7323) 2024-11-17T12:45:43,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741833_1009 (size=2288) 2024-11-17T12:45:43,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741833_1009 (size=2288) 2024-11-17T12:45:43,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741833_1009 (size=2288) 2024-11-17T12:45:43,735 DEBUG [RS:1;0d751a622496:44359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:45:43,735 INFO [RS:1;0d751a622496:44359 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T12:45:43,735 INFO [RS:1;0d751a622496:44359 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T12:45:43,736 INFO [RS:1;0d751a622496:44359 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-17T12:45:43,736 INFO [RS:1;0d751a622496:44359 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T12:45:43,736 INFO [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T12:45:43,736 INFO [RS:1;0d751a622496:44359 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44359 2024-11-17T12:45:43,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,44359,1731847526081 2024-11-17T12:45:43,788 INFO [RS:1;0d751a622496:44359 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T12:45:43,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T12:45:43,789 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,44359,1731847526081] 2024-11-17T12:45:43,804 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,44359,1731847526081 already deleted, retry=false 2024-11-17T12:45:43,804 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0d751a622496,44359,1731847526081 on 0d751a622496,38549,1731847525223 2024-11-17T12:45:43,809 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0d751a622496,44359,1731847526081, splitWal=true, meta=true 2024-11-17T12:45:43,812 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 0d751a622496,44359,1731847526081 (carryingMeta=true) 0d751a622496,44359,1731847526081/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1cbdb10c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-17T12:45:43,824 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0d751a622496,44359,1731847526081, splitWal=true, meta=true 2024-11-17T12:45:43,826 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 0d751a622496,44359,1731847526081, splitWal=true, meta=true, isMeta: true 2024-11-17T12:45:43,827 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting 2024-11-17T12:45:43,828 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0d751a622496,44359,1731847526081 WAL count=1, meta=true 2024-11-17T12:45:43,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta}] 2024-11-17T12:45:43,836 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=0d751a622496,38707,1731847525964 2024-11-17T12:45:43,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, worker=0d751a622496,38707,1731847525964}] 2024-11-17T12:45:43,850 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:43,851 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1] 2024-11-17T12:45:43,852 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0d751a622496:44359 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0d751a622496/172.17.0.2:44359 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:43,853 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1, error=java.net.ConnectException: Call to address=0d751a622496:44359 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0d751a622496/172.17.0.2:44359 2024-11-17T12:45:43,853 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-17T12:45:43,853 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 from cache 2024-11-17T12:45:43,853 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 0d751a622496:44359 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 0d751a622496/172.17.0.2:44359 2024-11-17T12:45:43,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:45:43,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44359-0x1014a1da31f0002, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:45:43,897 INFO [RS:1;0d751a622496:44359 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T12:45:43,897 INFO [RS:1;0d751a622496:44359 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,44359,1731847526081; zookeeper connection closed. 2024-11-17T12:45:43,897 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e0969c8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e0969c8 2024-11-17T12:45:44,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38707 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-17T12:45:44,023 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, size=7.2 K (7323bytes) 2024-11-17T12:45:44,023 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta 2024-11-17T12:45:44,024 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta after 1ms 2024-11-17T12:45:44,047 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:44,047 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta took 24ms 2024-11-17T12:45:44,054 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-17T12:45:44,054 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta so closing down 2024-11-17T12:45:44,054 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:44,056 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta.temp 2024-11-17T12:45:44,078 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta.temp 2024-11-17T12:45:44,078 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:44,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741892_1070 (size=7071) 2024-11-17T12:45:44,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741892_1070 (size=7071) 2024-11-17T12:45:44,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741892_1070 (size=7071) 2024-11-17T12:45:44,085 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:44,086 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta.temp to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-17T12:45:44,087 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-17T12:45:44,087 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, journal: Splitting hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, size=7.2 K (7323bytes) at 1731847544023Finishing writing output for hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta so closing down at 1731847544054 (+31 ms)Creating recovered edits writer path=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta.temp at 1731847544078 (+24 ms)3 split writer threads finished at 1731847544078Closed recovered edits writer path=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731847544085 (+7 ms)Rename recovered edits hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta.temp to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731847544086 (+1 ms)Processed 16 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1731847544087 (+1 ms) 2024-11-17T12:45:44,087 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta 2024-11-17T12:45:44,089 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-17T12:45:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-17T12:45:44,095 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs 2024-11-17T12:45:44,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-17T12:45:44,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, worker=0d751a622496,38707,1731847525964 in 257 msec 2024-11-17T12:45:44,099 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=0d751a622496,38707,1731847525964 2024-11-17T12:45:44,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T12:45:44,102 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 0d751a622496%2C44359%2C1731847526081.meta.1731847528091.meta, worker=0d751a622496,38707,1731847525964 in 270 msec 2024-11-17T12:45:44,104 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting dir is empty, no logs to split. 2024-11-17T12:45:44,104 INFO [PEWorker-1 {}] master.SplitWALManager(105): 0d751a622496,44359,1731847526081 WAL count=0, meta=true 2024-11-17T12:45:44,104 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 0d751a622496,44359,1731847526081 WAL splitting is done? wals=0, meta=true 2024-11-17T12:45:44,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T12:45:44,107 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T12:45:44,108 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T12:45:44,160 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:44,161 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1] 2024-11-17T12:45:44,161 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0d751a622496:44359 this server is in the failed servers list 2024-11-17T12:45:44,161 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0d751a622496:44359 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0d751a622496:44359 2024-11-17T12:45:44,162 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0d751a622496:44359 2024-11-17T12:45:44,162 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 from cache 2024-11-17T12:45:44,258 DEBUG [0d751a622496:38549 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-17T12:45:44,259 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(204): Hosts are {0d751a622496=0} racks are {/default-rack=0} 2024-11-17T12:45:44,259 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T12:45:44,259 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T12:45:44,259 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T12:45:44,259 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T12:45:44,259 INFO [0d751a622496:38549 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T12:45:44,259 INFO [0d751a622496:38549 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T12:45:44,259 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T12:45:44,259 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0d751a622496,38707,1731847525964 2024-11-17T12:45:44,260 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,38707,1731847525964, state=OPENING 2024-11-17T12:45:44,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:44,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:44,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:44,313 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T12:45:44,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:44,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:44,313 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:44,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,38707,1731847525964}] 2024-11-17T12:45:44,475 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T12:45:44,475 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:44,476 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T12:45:44,480 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0d751a622496%2C38707%2C1731847525964.meta, suffix=.meta, logDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,38707,1731847525964, archiveDir=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs, maxLogs=32 2024-11-17T12:45:44,495 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,38707,1731847525964/0d751a622496%2C38707%2C1731847525964.meta.1731847544480.meta, exclude list is [], retry=0 2024-11-17T12:45:44,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:44,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:44,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:44,520 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,38707,1731847525964/0d751a622496%2C38707%2C1731847525964.meta.1731847544480.meta 2024-11-17T12:45:44,520 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:44,520 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:44,521 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T12:45:44,521 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T12:45:44,521 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T12:45:44,521 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T12:45:44,521 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:44,521 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T12:45:44,521 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T12:45:44,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T12:45:44,523 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T12:45:44,523 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:44,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:44,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T12:45:44,525 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T12:45:44,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:44,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:44,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T12:45:44,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T12:45:44,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:44,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:44,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T12:45:44,527 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T12:45:44,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:44,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T12:45:44,528 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T12:45:44,529 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740 2024-11-17T12:45:44,530 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740 2024-11-17T12:45:44,530 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-17T12:45:44,555 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:44,558 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-17T12:45:44,558 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-17T12:45:44,575 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/info/c95dd66bbf7e40e38ff5c985368ffb36 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546./info:regioninfo/1731847543436/Put/seqid=0 2024-11-17T12:45:44,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741894_1072 (size=11177) 2024-11-17T12:45:44,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741894_1072 (size=11177) 2024-11-17T12:45:44,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741894_1072 (size=11177) 2024-11-17T12:45:44,581 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/info/c95dd66bbf7e40e38ff5c985368ffb36 2024-11-17T12:45:44,600 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/ns/1a245fe55c3946cbb1697f8ae8064638 is 43, key is default/ns:d/1731847528333/Put/seqid=0 2024-11-17T12:45:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741895_1073 (size=5153) 2024-11-17T12:45:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741895_1073 (size=5153) 2024-11-17T12:45:44,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741895_1073 (size=5153) 2024-11-17T12:45:44,607 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/ns/1a245fe55c3946cbb1697f8ae8064638 2024-11-17T12:45:44,625 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/table/32e76470436847308ae9b2d30fa2d107 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731847541703/Put/seqid=0 2024-11-17T12:45:44,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741896_1074 (size=5431) 2024-11-17T12:45:44,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741896_1074 (size=5431) 2024-11-17T12:45:44,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741896_1074 (size=5431) 2024-11-17T12:45:44,631 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/table/32e76470436847308ae9b2d30fa2d107 2024-11-17T12:45:44,638 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/info/c95dd66bbf7e40e38ff5c985368ffb36 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/info/c95dd66bbf7e40e38ff5c985368ffb36 2024-11-17T12:45:44,646 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/info/c95dd66bbf7e40e38ff5c985368ffb36, entries=36, sequenceid=18, filesize=10.9 K 2024-11-17T12:45:44,647 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/ns/1a245fe55c3946cbb1697f8ae8064638 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/ns/1a245fe55c3946cbb1697f8ae8064638 2024-11-17T12:45:44,654 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/ns/1a245fe55c3946cbb1697f8ae8064638, entries=2, sequenceid=18, filesize=5.0 K 2024-11-17T12:45:44,655 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/table/32e76470436847308ae9b2d30fa2d107 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/table/32e76470436847308ae9b2d30fa2d107 2024-11-17T12:45:44,661 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/table/32e76470436847308ae9b2d30fa2d107, entries=2, sequenceid=18, filesize=5.3 K 2024-11-17T12:45:44,661 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=18, compaction requested=false; wal=null 2024-11-17T12:45:44,662 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-17T12:45:44,664 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T12:45:44,664 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T12:45:44,664 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T12:45:44,665 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T12:45:44,668 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-17T12:45:44,669 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69807032, jitterRate=0.04020583629608154}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T12:45:44,669 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T12:45:44,669 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:44,670 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731847544521Writing region info on filesystem at 1731847544521Initializing all the Stores at 1731847544522 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847544522Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847544522Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847544522Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731847544522Obtaining lock to block concurrent updates at 1731847544558 (+36 ms)Preparing flush snapshotting stores in 1588230740 at 1731847544558Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731847544558Flushing stores of hbase:meta,,1.1588230740 at 1731847544558Flushing 1588230740/info: creating writer at 1731847544558Flushing 1588230740/info: appending metadata at 1731847544574 (+16 ms)Flushing 1588230740/info: closing flushed file at 1731847544574Flushing 1588230740/ns: creating writer at 1731847544587 (+13 ms)Flushing 1588230740/ns: appending metadata at 1731847544600 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731847544600Flushing 1588230740/table: creating writer at 1731847544612 (+12 ms)Flushing 1588230740/table: appending metadata at 1731847544624 (+12 ms)Flushing 1588230740/table: closing flushed file at 1731847544624Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74b8e82e: reopening flushed file at 1731847544637 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48c764a: reopening flushed file at 1731847544646 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11016e70: reopening flushed file at 1731847544654 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=18, compaction requested=false; wal=null at 1731847544661 (+7 ms)Cleaning up temporary data from old regions at 1731847544664 (+3 ms)Running coprocessor post-open hooks at 1731847544669 (+5 ms)Region opened successfully at 1731847544670 (+1 ms) 2024-11-17T12:45:44,671 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,38707,1731847525964, seqNum=-1] 2024-11-17T12:45:44,671 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731847544467 2024-11-17T12:45:44,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38707 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:54886 deadline: 1731847604671, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 0d751a622496,38707,1731847525964 2024-11-17T12:45:44,672 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,38707,1731847525964, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0d751a622496,38707,1731847525964, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 0d751a622496,38707,1731847525964 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T12:45:44,672 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,38707,1731847525964, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 0d751a622496,38707,1731847525964 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T12:45:44,672 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0d751a622496,38707,1731847525964, seqNum=-1 from cache 2024-11-17T12:45:44,673 DEBUG [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T12:45:44,673 INFO [RS_OPEN_META-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T12:45:44,674 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=0d751a622496,38707,1731847525964 2024-11-17T12:45:44,675 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0d751a622496,38707,1731847525964, state=OPEN 2024-11-17T12:45:44,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:44,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:44,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T12:45:44,687 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0d751a622496,38707,1731847525964 2024-11-17T12:45:44,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:44,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:44,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T12:45:44,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-17T12:45:44,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0d751a622496,38707,1731847525964 in 375 msec 2024-11-17T12:45:44,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-17T12:45:44,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 586 msec 2024-11-17T12:45:44,696 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 0d751a622496,44359,1731847526081 had 2 regions 2024-11-17T12:45:44,697 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0d751a622496,44359,1731847526081, splitWal=true, meta=true, isMeta: false 2024-11-17T12:45:44,699 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0d751a622496,44359,1731847526081 WAL count=1, meta=false 2024-11-17T12:45:44,699 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 0d751a622496%2C44359%2C1731847526081.1731847527698}] 2024-11-17T12:45:44,702 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=0d751a622496,38707,1731847525964 2024-11-17T12:45:44,703 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 0d751a622496%2C44359%2C1731847526081.1731847527698, worker=0d751a622496,38707,1731847525964}] 2024-11-17T12:45:44,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38707 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-17T12:45:44,876 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698, size=2.2 K (2288bytes) 2024-11-17T12:45:44,876 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698 2024-11-17T12:45:44,877 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698 after 1ms 2024-11-17T12:45:44,900 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:44,900 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698 took 24ms 2024-11-17T12:45:44,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 8e8bd106579494c8f12507d73483b546: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-17T12:45:44,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698 so closing down 2024-11-17T12:45:44,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:44,903 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:44,903 INFO [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-17T12:45:44,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698, journal: Splitting hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698, size=2.2 K (2288bytes) at 1731847544876Finishing writing output for hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698 so closing down at 1731847544903 (+27 ms)3 split writer threads finished at 1731847544903Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731847544903 2024-11-17T12:45:44,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698 2024-11-17T12:45:44,903 DEBUG [RS_LOG_REPLAY_OPS-regionserver/0d751a622496:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-17T12:45:44,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38549 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-17T12:45:44,907 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting/0d751a622496%2C44359%2C1731847526081.1731847527698 to hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs 2024-11-17T12:45:44,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-17T12:45:44,910 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 0d751a622496%2C44359%2C1731847526081.1731847527698, worker=0d751a622496,38707,1731847525964 in 204 msec 2024-11-17T12:45:44,912 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=0d751a622496,38707,1731847525964 2024-11-17T12:45:44,915 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-17T12:45:44,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 0d751a622496%2C44359%2C1731847526081.1731847527698, worker=0d751a622496,38707,1731847525964 in 213 msec 2024-11-17T12:45:44,917 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/WALs/0d751a622496,44359,1731847526081-splitting dir is empty, no logs to split. 2024-11-17T12:45:44,917 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0d751a622496,44359,1731847526081 WAL count=0, meta=false 2024-11-17T12:45:44,917 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0d751a622496,44359,1731847526081 WAL splitting is done? wals=0, meta=false 2024-11-17T12:45:44,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN}] 2024-11-17T12:45:44,921 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN 2024-11-17T12:45:44,922 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T12:45:45,072 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(204): Hosts are {0d751a622496=0} racks are {/default-rack=0} 2024-11-17T12:45:45,072 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T12:45:45,072 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T12:45:45,072 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T12:45:45,072 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T12:45:45,072 INFO [0d751a622496:38549 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T12:45:45,072 INFO [0d751a622496:38549 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T12:45:45,072 DEBUG [0d751a622496:38549 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T12:45:45,073 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPENING, regionLocation=0d751a622496,32849,1731847526132 2024-11-17T12:45:45,074 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0d751a622496:44359 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0d751a622496/172.17.0.2:44359 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:45,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1, error=java.net.ConnectException: Call to address=0d751a622496:44359 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0d751a622496/172.17.0.2:44359 2024-11-17T12:45:45,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-17T12:45:45,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0d751a622496,44359,1731847526081, seqNum=-1 from cache 2024-11-17T12:45:45,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 0d751a622496:44359 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0d751a622496/172.17.0.2:44359 2024-11-17T12:45:45,179 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:45,179 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,38707,1731847525964, seqNum=-1] 2024-11-17T12:45:45,180 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T12:45:45,181 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59585, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T12:45:45,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN because future has completed 2024-11-17T12:45:45,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,32849,1731847526132}] 2024-11-17T12:45:45,339 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T12:45:45,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58345, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T12:45:45,345 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:45,345 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 8e8bd106579494c8f12507d73483b546, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:45,345 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,345 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:45,345 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,345 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,347 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,348 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf1 2024-11-17T12:45:45,348 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:45,356 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf1/0108370a997c447a840348e1a160d410 2024-11-17T12:45:45,356 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:45,357 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,358 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8e8bd106579494c8f12507d73483b546 columnFamilyName cf2 2024-11-17T12:45:45,358 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:45,366 DEBUG [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/cf2/37e88620f6444cc198b9979272b7c2b5 2024-11-17T12:45:45,366 INFO [StoreOpener-8e8bd106579494c8f12507d73483b546-1 {}] regionserver.HStore(327): Store=8e8bd106579494c8f12507d73483b546/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:45,366 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,367 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,368 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,369 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,369 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,370 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-17T12:45:45,371 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,372 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 8e8bd106579494c8f12507d73483b546; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71017607, jitterRate=0.05824480950832367}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-17T12:45:45,372 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:45:45,373 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 8e8bd106579494c8f12507d73483b546: Running coprocessor pre-open hook at 1731847545346Writing region info on filesystem at 1731847545346Initializing all the Stores at 1731847545347 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847545347Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847545347Cleaning up temporary data from old regions at 1731847545369 (+22 ms)Running coprocessor post-open hooks at 1731847545372 (+3 ms)Region opened successfully at 1731847545373 (+1 ms) 2024-11-17T12:45:45,374 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., pid=21, masterSystemTime=1731847545338 2024-11-17T12:45:45,376 DEBUG [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:45,376 INFO [RS_OPEN_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:45:45,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=8e8bd106579494c8f12507d73483b546, regionState=OPEN, openSeqNum=18, regionLocation=0d751a622496,32849,1731847526132 2024-11-17T12:45:45,379 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,32849,1731847526132 because future has completed 2024-11-17T12:45:45,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-17T12:45:45,382 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 8e8bd106579494c8f12507d73483b546, server=0d751a622496,32849,1731847526132 in 196 msec 2024-11-17T12:45:45,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-17T12:45:45,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8e8bd106579494c8f12507d73483b546, ASSIGN in 462 msec 2024-11-17T12:45:45,384 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 0d751a622496,44359,1731847526081 after splitting done 2024-11-17T12:45:45,387 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 0d751a622496,44359,1731847526081, splitWal=true, meta=true in 1.5790 sec 2024-11-17T12:45:45,568 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-17T12:45:45,568 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:45,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-17T12:45:45,570 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:45,572 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-17T12:45:45,572 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:45,689 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T12:45:45,691 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0d751a622496,38707,1731847525964, seqNum=-1] 2024-11-17T12:45:45,810 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 0d751a622496:44359 this server is in the failed servers list 2024-11-17T12:45:45,810 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=0d751a622496:44359 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0d751a622496:44359 2024-11-17T12:45:45,810 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 0d751a622496:44359 2024-11-17T12:45:45,810 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,44359,1731847526081, seqNum=12 from cache 2024-11-17T12:45:46,033 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546., hostname=0d751a622496,32849,1731847526132, seqNum=18] 2024-11-17T12:45:46,033 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T12:45:46,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42230, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T12:45:46,048 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=404 (was 409), OpenFileDescriptor=1048 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 241), ProcessCount=11 (was 11), AvailableMemoryMB=6113 (was 6137) 2024-11-17T12:45:46,050 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-17T12:45:46,061 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=404, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=6111 2024-11-17T12:45:46,061 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-17T12:45:46,076 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:46,078 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:46,079 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:46,081 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99397742, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-99397742, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:46,092 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99397742/hregion-99397742.1731847546081, exclude list is [], retry=0 2024-11-17T12:45:46,095 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:46,095 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:46,095 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:46,118 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99397742/hregion-99397742.1731847546081 2024-11-17T12:45:46,118 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:46,119 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 310e375871342e21788c5904449fa1d9, NAME => 'testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741898_1076 (size=67) 2024-11-17T12:45:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741898_1076 (size=67) 2024-11-17T12:45:46,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741898_1076 (size=67) 2024-11-17T12:45:46,129 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:46,130 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,131 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName a 2024-11-17T12:45:46,131 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,132 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,132 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,133 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName b 2024-11-17T12:45:46,133 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,133 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,133 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,134 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName c 2024-11-17T12:45:46,135 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,135 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,135 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,136 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,136 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,137 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,137 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,137 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:46,138 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,140 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:46,140 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 310e375871342e21788c5904449fa1d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63545623, jitterRate=-0.053096428513526917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:46,141 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 310e375871342e21788c5904449fa1d9: Writing region info on filesystem at 1731847546129Initializing all the Stores at 1731847546129Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546130 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546130Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546130Cleaning up temporary data from old regions at 1731847546137 (+7 ms)Region opened successfully at 1731847546141 (+4 ms) 2024-11-17T12:45:46,141 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 310e375871342e21788c5904449fa1d9, disabling compactions & flushes 2024-11-17T12:45:46,141 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,141 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,141 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. after waiting 0 ms 2024-11-17T12:45:46,141 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,142 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,142 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 310e375871342e21788c5904449fa1d9: Waiting for close lock at 1731847546141Disabling compacts and flushes for region at 1731847546141Disabling writes for close at 1731847546141Writing region close event to WAL at 1731847546142 (+1 ms)Closed at 1731847546142 2024-11-17T12:45:46,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741897_1075 (size=175) 2024-11-17T12:45:46,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741897_1075 (size=175) 2024-11-17T12:45:46,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741897_1075 (size=175) 2024-11-17T12:45:46,147 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:46,147 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99397742:(num 1731847546081) 2024-11-17T12:45:46,147 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:46,148 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:46,161 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149, exclude list is [], retry=0 2024-11-17T12:45:46,163 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:46,163 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:46,164 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:46,187 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149 2024-11-17T12:45:46,188 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:46,188 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 310e375871342e21788c5904449fa1d9, NAME => 'testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:46,188 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:46,188 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,188 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,189 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,190 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName a 2024-11-17T12:45:46,190 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,191 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,191 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,192 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName b 2024-11-17T12:45:46,192 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,193 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,193 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,194 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName c 2024-11-17T12:45:46,194 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,194 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,194 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,195 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,196 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,197 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,197 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,198 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:46,199 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,200 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 310e375871342e21788c5904449fa1d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69105891, jitterRate=0.029758021235466003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:46,201 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 310e375871342e21788c5904449fa1d9: Writing region info on filesystem at 1731847546188Initializing all the Stores at 1731847546189 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546189Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546189Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546189Cleaning up temporary data from old regions at 1731847546198 (+9 ms)Region opened successfully at 1731847546200 (+2 ms) 2024-11-17T12:45:46,228 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 310e375871342e21788c5904449fa1d9 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-17T12:45:46,248 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/a/2411dd59674941ab95fa551e4e29e153 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731847546201/Put/seqid=0 2024-11-17T12:45:46,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741900_1078 (size=5958) 2024-11-17T12:45:46,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741900_1078 (size=5958) 2024-11-17T12:45:46,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741900_1078 (size=5958) 2024-11-17T12:45:46,255 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/a/2411dd59674941ab95fa551e4e29e153 2024-11-17T12:45:46,274 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/b/fc0c4a5a044b491fa3408136adc50dbd is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731847546209/Put/seqid=0 2024-11-17T12:45:46,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741901_1079 (size=5958) 2024-11-17T12:45:46,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741901_1079 (size=5958) 2024-11-17T12:45:46,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741901_1079 (size=5958) 2024-11-17T12:45:46,281 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/b/fc0c4a5a044b491fa3408136adc50dbd 2024-11-17T12:45:46,299 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/c/b7d7a3ce5ec74b469b5a9ef5766e0ccc is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731847546218/Put/seqid=0 2024-11-17T12:45:46,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741902_1080 (size=5958) 2024-11-17T12:45:46,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741902_1080 (size=5958) 2024-11-17T12:45:46,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741902_1080 (size=5958) 2024-11-17T12:45:46,308 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/c/b7d7a3ce5ec74b469b5a9ef5766e0ccc 2024-11-17T12:45:46,314 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/a/2411dd59674941ab95fa551e4e29e153 as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/a/2411dd59674941ab95fa551e4e29e153 2024-11-17T12:45:46,320 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/a/2411dd59674941ab95fa551e4e29e153, entries=10, sequenceid=33, filesize=5.8 K 2024-11-17T12:45:46,322 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/b/fc0c4a5a044b491fa3408136adc50dbd as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/b/fc0c4a5a044b491fa3408136adc50dbd 2024-11-17T12:45:46,330 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/b/fc0c4a5a044b491fa3408136adc50dbd, entries=10, sequenceid=33, filesize=5.8 K 2024-11-17T12:45:46,331 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/c/b7d7a3ce5ec74b469b5a9ef5766e0ccc as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/c/b7d7a3ce5ec74b469b5a9ef5766e0ccc 2024-11-17T12:45:46,337 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/c/b7d7a3ce5ec74b469b5a9ef5766e0ccc, entries=10, sequenceid=33, filesize=5.8 K 2024-11-17T12:45:46,338 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 310e375871342e21788c5904449fa1d9 in 110ms, sequenceid=33, compaction requested=false 2024-11-17T12:45:46,338 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 310e375871342e21788c5904449fa1d9: 2024-11-17T12:45:46,338 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 310e375871342e21788c5904449fa1d9, disabling compactions & flushes 2024-11-17T12:45:46,338 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,338 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,339 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. after waiting 0 ms 2024-11-17T12:45:46,339 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,340 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. 2024-11-17T12:45:46,340 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 310e375871342e21788c5904449fa1d9: Waiting for close lock at 1731847546338Disabling compacts and flushes for region at 1731847546338Disabling writes for close at 1731847546339 (+1 ms)Writing region close event to WAL at 1731847546340 (+1 ms)Closed at 1731847546340 2024-11-17T12:45:46,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741899_1077 (size=6311) 2024-11-17T12:45:46,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741899_1077 (size=6311) 2024-11-17T12:45:46,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741899_1077 (size=6311) 2024-11-17T12:45:46,347 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/b/fc0c4a5a044b491fa3408136adc50dbd to hdfs://localhost:37973/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/b/fc0c4a5a044b491fa3408136adc50dbd 2024-11-17T12:45:46,370 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149, size=6.2 K (6311bytes) 2024-11-17T12:45:46,370 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149 2024-11-17T12:45:46,371 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149 after 1ms 2024-11-17T12:45:46,398 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:46,398 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149 took 28ms 2024-11-17T12:45:46,400 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149 so closing down 2024-11-17T12:45:46,401 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:46,401 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731847546149.temp 2024-11-17T12:45:46,427 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000003-wal.1731847546149.temp 2024-11-17T12:45:46,427 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:46,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741903_1081 (size=5575) 2024-11-17T12:45:46,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741903_1081 (size=5575) 2024-11-17T12:45:46,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741903_1081 (size=5575) 2024-11-17T12:45:46,841 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000003-wal.1731847546149.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:46,845 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000003-wal.1731847546149.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000032 2024-11-17T12:45:46,845 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 447 ms; skipped=2; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-17T12:45:46,845 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149, journal: Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149, size=6.2 K (6311bytes) at 1731847546370Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149 so closing down at 1731847546400 (+30 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000003-wal.1731847546149.temp at 1731847546427 (+27 ms)3 split writer threads finished at 1731847546427Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000003-wal.1731847546149.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731847546841 (+414 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000003-wal.1731847546149.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000032 at 1731847546845 (+4 ms)Processed 32 edits across 1 Regions in 447 ms; skipped=2; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731847546845 2024-11-17T12:45:46,849 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546149 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847546149 2024-11-17T12:45:46,850 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000032 2024-11-17T12:45:46,851 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:46,853 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:46,867 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546853, exclude list is [], retry=0 2024-11-17T12:45:46,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:46,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:46,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:46,892 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546853 2024-11-17T12:45:46,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:46,893 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 310e375871342e21788c5904449fa1d9, NAME => 'testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:46,893 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:46,893 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,893 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,894 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,895 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName a 2024-11-17T12:45:46,895 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,900 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/a/2411dd59674941ab95fa551e4e29e153 2024-11-17T12:45:46,901 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,901 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,902 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName b 2024-11-17T12:45:46,902 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,903 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,903 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,904 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 310e375871342e21788c5904449fa1d9 columnFamilyName c 2024-11-17T12:45:46,904 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:46,911 DEBUG [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/c/b7d7a3ce5ec74b469b5a9ef5766e0ccc 2024-11-17T12:45:46,911 INFO [StoreOpener-310e375871342e21788c5904449fa1d9-1 {}] regionserver.HStore(327): Store=310e375871342e21788c5904449fa1d9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:46,911 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,914 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000032 2024-11-17T12:45:46,938 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:46,939 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000032 2024-11-17T12:45:46,940 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 310e375871342e21788c5904449fa1d9 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-17T12:45:46,952 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/b/d3934385d7634aa8b70af49551134722 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731847546209/Put/seqid=0 2024-11-17T12:45:46,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741905_1083 (size=5958) 2024-11-17T12:45:46,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741905_1083 (size=5958) 2024-11-17T12:45:46,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741905_1083 (size=5958) 2024-11-17T12:45:46,959 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/b/d3934385d7634aa8b70af49551134722 2024-11-17T12:45:46,965 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/.tmp/b/d3934385d7634aa8b70af49551134722 as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/b/d3934385d7634aa8b70af49551134722 2024-11-17T12:45:46,970 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/b/d3934385d7634aa8b70af49551134722, entries=10, sequenceid=32, filesize=5.8 K 2024-11-17T12:45:46,970 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 310e375871342e21788c5904449fa1d9 in 30ms, sequenceid=32, compaction requested=false; wal=null 2024-11-17T12:45:46,971 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/0000000000000000032 2024-11-17T12:45:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,972 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,973 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:46,974 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 310e375871342e21788c5904449fa1d9 2024-11-17T12:45:46,977 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/310e375871342e21788c5904449fa1d9/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-17T12:45:46,978 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 310e375871342e21788c5904449fa1d9; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74711947, jitterRate=0.11329476535320282}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:46,978 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 310e375871342e21788c5904449fa1d9: Writing region info on filesystem at 1731847546893Initializing all the Stores at 1731847546894 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546894Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546894Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847546894Obtaining lock to block concurrent updates at 1731847546940 (+46 ms)Preparing flush snapshotting stores in 310e375871342e21788c5904449fa1d9 at 1731847546940Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731847546940Flushing stores of testReplayEditsWrittenViaHRegion,,1731847546077.310e375871342e21788c5904449fa1d9. at 1731847546940Flushing 310e375871342e21788c5904449fa1d9/b: creating writer at 1731847546940Flushing 310e375871342e21788c5904449fa1d9/b: appending metadata at 1731847546952 (+12 ms)Flushing 310e375871342e21788c5904449fa1d9/b: closing flushed file at 1731847546952Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ab5265f: reopening flushed file at 1731847546964 (+12 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 310e375871342e21788c5904449fa1d9 in 30ms, sequenceid=32, compaction requested=false; wal=null at 1731847546970 (+6 ms)Cleaning up temporary data from old regions at 1731847546972 (+2 ms)Region opened successfully at 1731847546978 (+6 ms) 2024-11-17T12:45:46,997 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=413 (was 404) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:45142 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:39170 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:45156 [Waiting for operation #48] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:50230 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:56412 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:45314 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1116 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=230 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=6100 (was 6111) 2024-11-17T12:45:46,997 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-17T12:45:47,007 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413, OpenFileDescriptor=1116, MaxFileDescriptor=1048576, SystemLoadAverage=230, ProcessCount=11, AvailableMemoryMB=6100 2024-11-17T12:45:47,007 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-17T12:45:47,020 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:47,022 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:47,022 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:47,025 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-10158186, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-10158186, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:47,036 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-10158186/hregion-10158186.1731847547025, exclude list is [], retry=0 2024-11-17T12:45:47,038 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:47,038 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:47,039 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:47,061 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-10158186/hregion-10158186.1731847547025 2024-11-17T12:45:47,061 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:47,061 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4489bd2d11ba8673ac487083625981b6, NAME => 'testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741907_1085 (size=68) 2024-11-17T12:45:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741907_1085 (size=68) 2024-11-17T12:45:47,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741907_1085 (size=68) 2024-11-17T12:45:47,069 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:47,070 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,071 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName a 2024-11-17T12:45:47,071 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:47,071 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:47,071 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,073 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName b 2024-11-17T12:45:47,073 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:47,073 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:47,073 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,074 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName c 2024-11-17T12:45:47,075 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:47,075 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:47,075 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,077 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,078 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:47,079 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,081 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:47,082 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4489bd2d11ba8673ac487083625981b6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73660476, jitterRate=0.09762662649154663}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4489bd2d11ba8673ac487083625981b6: Writing region info on filesystem at 1731847547069Initializing all the Stores at 1731847547069Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847547069Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847547070 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847547070Cleaning up temporary data from old regions at 1731847547077 (+7 ms)Region opened successfully at 1731847547083 (+6 ms) 2024-11-17T12:45:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4489bd2d11ba8673ac487083625981b6, disabling compactions & flushes 2024-11-17T12:45:47,083 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. after waiting 0 ms 2024-11-17T12:45:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:47,083 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:47,083 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4489bd2d11ba8673ac487083625981b6: Waiting for close lock at 1731847547083Disabling compacts and flushes for region at 1731847547083Disabling writes for close at 1731847547083Writing region close event to WAL at 1731847547083Closed at 1731847547083 2024-11-17T12:45:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741906_1084 (size=175) 2024-11-17T12:45:47,087 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-10158186/hregion-10158186.1731847547025 not finished, retry = 0 2024-11-17T12:45:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741906_1084 (size=175) 2024-11-17T12:45:47,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741906_1084 (size=175) 2024-11-17T12:45:47,190 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:47,190 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-10158186:(num 1731847547025) 2024-11-17T12:45:47,190 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:47,192 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:47,203 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192, exclude list is [], retry=0 2024-11-17T12:45:47,206 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:47,207 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:47,207 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:47,208 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T12:45:47,237 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192 2024-11-17T12:45:47,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:47,309 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4489bd2d11ba8673ac487083625981b6, NAME => 'testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:47,311 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,311 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:47,311 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,311 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,313 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,314 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName a 2024-11-17T12:45:47,314 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:47,314 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:47,315 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,315 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName b 2024-11-17T12:45:47,315 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:47,316 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:47,316 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,317 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName c 2024-11-17T12:45:47,317 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:47,317 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:47,317 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,318 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,319 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,320 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,320 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,320 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:47,321 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,322 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4489bd2d11ba8673ac487083625981b6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68336675, jitterRate=0.018295809626579285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:47,322 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:47,323 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4489bd2d11ba8673ac487083625981b6: Running coprocessor pre-open hook at 1731847547311Writing region info on filesystem at 1731847547311Initializing all the Stores at 1731847547312 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847547312Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847547312Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847547312Cleaning up temporary data from old regions at 1731847547320 (+8 ms)Running coprocessor post-open hooks at 1731847547322 (+2 ms)Region opened successfully at 1731847547323 (+1 ms) 2024-11-17T12:45:47,332 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4489bd2d11ba8673ac487083625981b6 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-17T12:45:47,333 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:48,334 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:49,335 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:50,337 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:51,338 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:52,339 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:52,813 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T12:45:53,339 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:54,340 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:54,377 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T12:45:55,341 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:55,568 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-17T12:45:55,568 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:55,569 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-17T12:45:55,569 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-17T12:45:56,342 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 4489bd2d11ba8673ac487083625981b6/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:45:56,343 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4489bd2d11ba8673ac487083625981b6: 2024-11-17T12:45:56,343 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:56,355 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4489bd2d11ba8673ac487083625981b6: 2024-11-17T12:45:56,355 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-17T12:45:56,355 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4489bd2d11ba8673ac487083625981b6, disabling compactions & flushes 2024-11-17T12:45:56,356 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:56,356 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:56,356 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. after waiting 0 ms 2024-11-17T12:45:56,356 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:56,356 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:56,356 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. 2024-11-17T12:45:56,356 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4489bd2d11ba8673ac487083625981b6: Waiting for close lock at 1731847556355Running coprocessor pre-close hooks at 1731847556355Disabling compacts and flushes for region at 1731847556355Disabling writes for close at 1731847556356 (+1 ms)Writing region close event to WAL at 1731847556356Running coprocessor post-close hooks at 1731847556356Closed at 1731847556356 2024-11-17T12:45:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741908_1086 (size=4236) 2024-11-17T12:45:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741908_1086 (size=4236) 2024-11-17T12:45:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741908_1086 (size=4236) 2024-11-17T12:45:56,372 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192, size=4.1 K (4236bytes) 2024-11-17T12:45:56,372 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192 2024-11-17T12:45:56,373 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192 after 1ms 2024-11-17T12:45:56,397 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:56,397 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192 took 25ms 2024-11-17T12:45:56,399 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192 so closing down 2024-11-17T12:45:56,399 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:56,400 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731847547192.temp 2024-11-17T12:45:56,422 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000004-wal.1731847547192.temp 2024-11-17T12:45:56,422 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:56,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741909_1087 (size=3245) 2024-11-17T12:45:56,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741909_1087 (size=3245) 2024-11-17T12:45:56,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741909_1087 (size=3245) 2024-11-17T12:45:56,428 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000004-wal.1731847547192.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:56,429 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000004-wal.1731847547192.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000026 2024-11-17T12:45:56,429 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 32 ms; skipped=3; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-17T12:45:56,429 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192, journal: Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192, size=4.1 K (4236bytes) at 1731847556372Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192 so closing down at 1731847556399 (+27 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000004-wal.1731847547192.temp at 1731847556422 (+23 ms)3 split writer threads finished at 1731847556422Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000004-wal.1731847547192.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731847556428 (+6 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000004-wal.1731847547192.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000026 at 1731847556429 (+1 ms)Processed 23 edits across 1 Regions in 32 ms; skipped=3; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731847556429 2024-11-17T12:45:56,431 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847547192 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847547192 2024-11-17T12:45:56,432 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000026 2024-11-17T12:45:56,432 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:56,434 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:56,448 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847556434, exclude list is [], retry=0 2024-11-17T12:45:56,450 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:56,450 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:56,451 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:56,473 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847556434 2024-11-17T12:45:56,474 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:56,474 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4489bd2d11ba8673ac487083625981b6, NAME => 'testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:56,474 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,474 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:56,475 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,475 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,476 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,477 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName a 2024-11-17T12:45:56,477 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:56,478 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:56,478 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,479 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName b 2024-11-17T12:45:56,479 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:56,479 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:56,480 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,480 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4489bd2d11ba8673ac487083625981b6 columnFamilyName c 2024-11-17T12:45:56,480 DEBUG [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:56,481 INFO [StoreOpener-4489bd2d11ba8673ac487083625981b6-1 {}] regionserver.HStore(327): Store=4489bd2d11ba8673ac487083625981b6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:56,481 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,482 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,483 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,484 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000026 2024-11-17T12:45:56,508 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:56,510 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000026 2024-11-17T12:45:56,510 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4489bd2d11ba8673ac487083625981b6 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-17T12:45:56,523 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/a/b84f8be19ccb459284db01d0f99748d8 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731847556347/Put/seqid=0 2024-11-17T12:45:56,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741911_1089 (size=5523) 2024-11-17T12:45:56,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741911_1089 (size=5523) 2024-11-17T12:45:56,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741911_1089 (size=5523) 2024-11-17T12:45:56,530 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/a/b84f8be19ccb459284db01d0f99748d8 2024-11-17T12:45:56,548 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/b/1f7a6f2ae5e2471a886b7fd5332b7031 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731847556343/Put/seqid=0 2024-11-17T12:45:56,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741912_1090 (size=5524) 2024-11-17T12:45:56,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741912_1090 (size=5524) 2024-11-17T12:45:56,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741912_1090 (size=5524) 2024-11-17T12:45:56,554 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/b/1f7a6f2ae5e2471a886b7fd5332b7031 2024-11-17T12:45:56,571 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/c/305a807194904b5ba1dd106860d90356 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731847556345/Put/seqid=0 2024-11-17T12:45:56,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741913_1091 (size=5457) 2024-11-17T12:45:56,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741913_1091 (size=5457) 2024-11-17T12:45:56,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741913_1091 (size=5457) 2024-11-17T12:45:56,577 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/c/305a807194904b5ba1dd106860d90356 2024-11-17T12:45:56,582 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/a/b84f8be19ccb459284db01d0f99748d8 as hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/a/b84f8be19ccb459284db01d0f99748d8 2024-11-17T12:45:56,587 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/a/b84f8be19ccb459284db01d0f99748d8, entries=7, sequenceid=26, filesize=5.4 K 2024-11-17T12:45:56,588 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/b/1f7a6f2ae5e2471a886b7fd5332b7031 as hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/b/1f7a6f2ae5e2471a886b7fd5332b7031 2024-11-17T12:45:56,593 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/b/1f7a6f2ae5e2471a886b7fd5332b7031, entries=7, sequenceid=26, filesize=5.4 K 2024-11-17T12:45:56,594 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/.tmp/c/305a807194904b5ba1dd106860d90356 as hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/c/305a807194904b5ba1dd106860d90356 2024-11-17T12:45:56,599 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/c/305a807194904b5ba1dd106860d90356, entries=6, sequenceid=26, filesize=5.3 K 2024-11-17T12:45:56,600 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 4489bd2d11ba8673ac487083625981b6 in 89ms, sequenceid=26, compaction requested=false; wal=null 2024-11-17T12:45:56,600 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/0000000000000000026 2024-11-17T12:45:56,601 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,601 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,602 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:56,603 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,605 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsAfterAbortingFlush/4489bd2d11ba8673ac487083625981b6/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-17T12:45:56,605 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4489bd2d11ba8673ac487083625981b6; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60664136, jitterRate=-0.09603393077850342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:56,606 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4489bd2d11ba8673ac487083625981b6 2024-11-17T12:45:56,606 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4489bd2d11ba8673ac487083625981b6: Running coprocessor pre-open hook at 1731847556475Writing region info on filesystem at 1731847556475Initializing all the Stores at 1731847556476 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847556476Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847556476Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847556476Obtaining lock to block concurrent updates at 1731847556510 (+34 ms)Preparing flush snapshotting stores in 4489bd2d11ba8673ac487083625981b6 at 1731847556510Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731847556510Flushing stores of testReplayEditsAfterAbortingFlush,,1731847547021.4489bd2d11ba8673ac487083625981b6. at 1731847556510Flushing 4489bd2d11ba8673ac487083625981b6/a: creating writer at 1731847556510Flushing 4489bd2d11ba8673ac487083625981b6/a: appending metadata at 1731847556523 (+13 ms)Flushing 4489bd2d11ba8673ac487083625981b6/a: closing flushed file at 1731847556523Flushing 4489bd2d11ba8673ac487083625981b6/b: creating writer at 1731847556535 (+12 ms)Flushing 4489bd2d11ba8673ac487083625981b6/b: appending metadata at 1731847556547 (+12 ms)Flushing 4489bd2d11ba8673ac487083625981b6/b: closing flushed file at 1731847556547Flushing 4489bd2d11ba8673ac487083625981b6/c: creating writer at 1731847556559 (+12 ms)Flushing 4489bd2d11ba8673ac487083625981b6/c: appending metadata at 1731847556570 (+11 ms)Flushing 4489bd2d11ba8673ac487083625981b6/c: closing flushed file at 1731847556570Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a9d7b0a: reopening flushed file at 1731847556581 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75439c00: reopening flushed file at 1731847556587 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dbbec1b: reopening flushed file at 1731847556593 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 4489bd2d11ba8673ac487083625981b6 in 89ms, sequenceid=26, compaction requested=false; wal=null at 1731847556600 (+7 ms)Cleaning up temporary data from old regions at 1731847556601 (+1 ms)Running coprocessor post-open hooks at 1731847556606 (+5 ms)Region opened successfully at 1731847556606 2024-11-17T12:45:56,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 413) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:41044 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:34346 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:52374 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:34364 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:52364 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:41040 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1116) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=195 (was 230), ProcessCount=11 (was 11), AvailableMemoryMB=6067 (was 6100) 2024-11-17T12:45:56,626 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-17T12:45:56,638 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=195, ProcessCount=11, AvailableMemoryMB=6066 2024-11-17T12:45:56,638 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-17T12:45:56,655 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:56,656 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:56,657 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:56,659 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-86755301, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-86755301, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:56,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-86755301/hregion-86755301.1731847556660, exclude list is [], retry=0 2024-11-17T12:45:56,673 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:56,673 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:56,673 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:56,697 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-86755301/hregion-86755301.1731847556660 2024-11-17T12:45:56,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:56,698 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0fa61ae66a89fd32b31ad4774107345d, NAME => 'testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741915_1093 (size=61) 2024-11-17T12:45:56,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741915_1093 (size=61) 2024-11-17T12:45:56,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741915_1093 (size=61) 2024-11-17T12:45:56,709 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:56,710 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,712 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa61ae66a89fd32b31ad4774107345d columnFamilyName a 2024-11-17T12:45:56,712 DEBUG [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:56,712 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(327): Store=0fa61ae66a89fd32b31ad4774107345d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:56,713 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,713 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,713 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,714 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,714 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,715 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,717 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:56,718 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0fa61ae66a89fd32b31ad4774107345d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63416417, jitterRate=-0.0550217479467392}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:56,718 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0fa61ae66a89fd32b31ad4774107345d: Writing region info on filesystem at 1731847556709Initializing all the Stores at 1731847556710 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847556710Cleaning up temporary data from old regions at 1731847556714 (+4 ms)Region opened successfully at 1731847556718 (+4 ms) 2024-11-17T12:45:56,718 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0fa61ae66a89fd32b31ad4774107345d, disabling compactions & flushes 2024-11-17T12:45:56,718 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,718 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,718 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. after waiting 0 ms 2024-11-17T12:45:56,718 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,719 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,719 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0fa61ae66a89fd32b31ad4774107345d: Waiting for close lock at 1731847556718Disabling compacts and flushes for region at 1731847556718Disabling writes for close at 1731847556718Writing region close event to WAL at 1731847556719 (+1 ms)Closed at 1731847556719 2024-11-17T12:45:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741914_1092 (size=175) 2024-11-17T12:45:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741914_1092 (size=175) 2024-11-17T12:45:56,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741914_1092 (size=175) 2024-11-17T12:45:56,724 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:56,724 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-86755301:(num 1731847556660) 2024-11-17T12:45:56,724 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:56,725 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:56,736 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726, exclude list is [], retry=0 2024-11-17T12:45:56,739 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:56,739 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:56,739 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:56,763 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726 2024-11-17T12:45:56,763 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:56,763 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fa61ae66a89fd32b31ad4774107345d, NAME => 'testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:56,763 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:56,763 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,763 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,765 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,765 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa61ae66a89fd32b31ad4774107345d columnFamilyName a 2024-11-17T12:45:56,765 DEBUG [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:56,766 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(327): Store=0fa61ae66a89fd32b31ad4774107345d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:56,766 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,767 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,768 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,768 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,768 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,769 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:56,770 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0fa61ae66a89fd32b31ad4774107345d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63562445, jitterRate=-0.05284576117992401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:56,770 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0fa61ae66a89fd32b31ad4774107345d: Writing region info on filesystem at 1731847556764Initializing all the Stores at 1731847556764Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847556764Cleaning up temporary data from old regions at 1731847556768 (+4 ms)Region opened successfully at 1731847556770 (+2 ms) 2024-11-17T12:45:56,779 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0fa61ae66a89fd32b31ad4774107345d, disabling compactions & flushes 2024-11-17T12:45:56,779 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,779 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,779 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. after waiting 0 ms 2024-11-17T12:45:56,779 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,780 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,780 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. 2024-11-17T12:45:56,780 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0fa61ae66a89fd32b31ad4774107345d: Waiting for close lock at 1731847556779Disabling compacts and flushes for region at 1731847556779Disabling writes for close at 1731847556779Writing region close event to WAL at 1731847556780 (+1 ms)Closed at 1731847556780 2024-11-17T12:45:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741916_1094 (size=1795) 2024-11-17T12:45:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741916_1094 (size=1795) 2024-11-17T12:45:56,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741916_1094 (size=1795) 2024-11-17T12:45:56,798 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726, size=1.8 K (1795bytes) 2024-11-17T12:45:56,799 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726 2024-11-17T12:45:56,799 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726 after 0ms 2024-11-17T12:45:56,822 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:56,822 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726 took 24ms 2024-11-17T12:45:56,824 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726 so closing down 2024-11-17T12:45:56,824 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:45:56,825 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731847556726.temp 2024-11-17T12:45:56,847 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000003-wal.1731847556726.temp 2024-11-17T12:45:56,848 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:45:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741917_1095 (size=1795) 2024-11-17T12:45:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741917_1095 (size=1795) 2024-11-17T12:45:56,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741917_1095 (size=1795) 2024-11-17T12:45:56,854 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000003-wal.1731847556726.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-17T12:45:56,856 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000003-wal.1731847556726.temp to hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000012 2024-11-17T12:45:56,856 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-17T12:45:56,856 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726, journal: Splitting hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726, size=1.8 K (1795bytes) at 1731847556798Finishing writing output for hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726 so closing down at 1731847556824 (+26 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000003-wal.1731847556726.temp at 1731847556847 (+23 ms)3 split writer threads finished at 1731847556848 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000003-wal.1731847556726.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731847556854 (+6 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000003-wal.1731847556726.temp to hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000012 at 1731847556856 (+2 ms)Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731847556856 2024-11-17T12:45:56,857 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847556726 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847556726 2024-11-17T12:45:56,858 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000012 2024-11-17T12:45:56,884 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:57,233 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:57,235 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:57,246 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847557235, exclude list is [], retry=0 2024-11-17T12:45:57,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:57,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:57,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:57,275 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847557235 2024-11-17T12:45:57,275 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:57,275 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fa61ae66a89fd32b31ad4774107345d, NAME => 'testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:57,276 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:57,276 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,276 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,278 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,280 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa61ae66a89fd32b31ad4774107345d columnFamilyName a 2024-11-17T12:45:57,280 DEBUG [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,282 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(327): Store=0fa61ae66a89fd32b31ad4774107345d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,282 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,283 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,284 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,285 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000012 2024-11-17T12:45:57,310 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:45:57,311 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000012 2024-11-17T12:45:57,311 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0fa61ae66a89fd32b31ad4774107345d 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-17T12:45:57,327 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/.tmp/a/422ae8867fbb41f19c1e3e5c75778aaa is 79, key is testDatalossWhenInputError/a:x0/1731847556771/Put/seqid=0 2024-11-17T12:45:57,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741919_1097 (size=5808) 2024-11-17T12:45:57,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741919_1097 (size=5808) 2024-11-17T12:45:57,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741919_1097 (size=5808) 2024-11-17T12:45:57,332 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/.tmp/a/422ae8867fbb41f19c1e3e5c75778aaa 2024-11-17T12:45:57,341 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/.tmp/a/422ae8867fbb41f19c1e3e5c75778aaa as hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/a/422ae8867fbb41f19c1e3e5c75778aaa 2024-11-17T12:45:57,349 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/a/422ae8867fbb41f19c1e3e5c75778aaa, entries=10, sequenceid=12, filesize=5.7 K 2024-11-17T12:45:57,350 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 0fa61ae66a89fd32b31ad4774107345d in 39ms, sequenceid=12, compaction requested=false; wal=null 2024-11-17T12:45:57,351 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/0000000000000000012 2024-11-17T12:45:57,352 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,352 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,354 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,356 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-17T12:45:57,357 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0fa61ae66a89fd32b31ad4774107345d; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64435464, jitterRate=-0.039836764335632324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:57,357 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0fa61ae66a89fd32b31ad4774107345d: Writing region info on filesystem at 1731847557276Initializing all the Stores at 1731847557278 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557278Obtaining lock to block concurrent updates at 1731847557311 (+33 ms)Preparing flush snapshotting stores in 0fa61ae66a89fd32b31ad4774107345d at 1731847557311Finished memstore snapshotting testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731847557311Flushing stores of testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d. at 1731847557311Flushing 0fa61ae66a89fd32b31ad4774107345d/a: creating writer at 1731847557312 (+1 ms)Flushing 0fa61ae66a89fd32b31ad4774107345d/a: appending metadata at 1731847557326 (+14 ms)Flushing 0fa61ae66a89fd32b31ad4774107345d/a: closing flushed file at 1731847557326Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7591ff6d: reopening flushed file at 1731847557339 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 0fa61ae66a89fd32b31ad4774107345d in 39ms, sequenceid=12, compaction requested=false; wal=null at 1731847557350 (+11 ms)Cleaning up temporary data from old regions at 1731847557352 (+2 ms)Region opened successfully at 1731847557357 (+5 ms) 2024-11-17T12:45:57,360 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fa61ae66a89fd32b31ad4774107345d, NAME => 'testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:57,360 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731847556655.0fa61ae66a89fd32b31ad4774107345d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:57,361 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,361 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,362 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,363 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fa61ae66a89fd32b31ad4774107345d columnFamilyName a 2024-11-17T12:45:57,363 DEBUG [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,367 DEBUG [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/a/422ae8867fbb41f19c1e3e5c75778aaa 2024-11-17T12:45:57,368 INFO [StoreOpener-0fa61ae66a89fd32b31ad4774107345d-1 {}] regionserver.HStore(327): Store=0fa61ae66a89fd32b31ad4774107345d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,368 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,368 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,371 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0fa61ae66a89fd32b31ad4774107345d 2024-11-17T12:45:57,374 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testDatalossWhenInputError/0fa61ae66a89fd32b31ad4774107345d/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-17T12:45:57,375 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0fa61ae66a89fd32b31ad4774107345d; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68545191, jitterRate=0.021402940154075623}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T12:45:57,375 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0fa61ae66a89fd32b31ad4774107345d: Writing region info on filesystem at 1731847557361Initializing all the Stores at 1731847557361Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557361Cleaning up temporary data from old regions at 1731847557370 (+9 ms)Region opened successfully at 1731847557375 (+5 ms) 2024-11-17T12:45:57,390 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=425 (was 415) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:41044 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:34424 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-131233367-172.17.0.2-1731847521320:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:34346 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:52374 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:41122 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:52412 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=195 (was 195), ProcessCount=11 (was 11), AvailableMemoryMB=6056 (was 6066) 2024-11-17T12:45:57,390 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-17T12:45:57,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=425, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=195, ProcessCount=11, AvailableMemoryMB=6055 2024-11-17T12:45:57,404 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-17T12:45:57,421 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:57,422 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:45:57,423 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:45:57,425 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-75384987, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-75384987, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:57,435 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-75384987/hregion-75384987.1731847557425, exclude list is [], retry=0 2024-11-17T12:45:57,438 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:57,438 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:57,438 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:57,462 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-75384987/hregion-75384987.1731847557425 2024-11-17T12:45:57,462 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:45:57,462 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => c92256f72a75329f061423749b98ffaf, NAME => 'testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:45:57,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741921_1099 (size=63) 2024-11-17T12:45:57,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741921_1099 (size=63) 2024-11-17T12:45:57,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741921_1099 (size=63) 2024-11-17T12:45:57,471 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:57,473 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,474 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName a 2024-11-17T12:45:57,474 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,474 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,474 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,476 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName b 2024-11-17T12:45:57,476 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,476 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,476 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,477 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName c 2024-11-17T12:45:57,477 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,478 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,478 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,478 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,479 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,480 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,480 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,480 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:57,481 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,483 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:45:57,484 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c92256f72a75329f061423749b98ffaf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70949932, jitterRate=0.05723637342453003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:57,484 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c92256f72a75329f061423749b98ffaf: Writing region info on filesystem at 1731847557472Initializing all the Stores at 1731847557472Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557472Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557472Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557472Cleaning up temporary data from old regions at 1731847557480 (+8 ms)Region opened successfully at 1731847557484 (+4 ms) 2024-11-17T12:45:57,484 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c92256f72a75329f061423749b98ffaf, disabling compactions & flushes 2024-11-17T12:45:57,484 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:45:57,484 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:45:57,484 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. after waiting 0 ms 2024-11-17T12:45:57,484 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:45:57,485 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:45:57,485 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c92256f72a75329f061423749b98ffaf: Waiting for close lock at 1731847557484Disabling compacts and flushes for region at 1731847557484Disabling writes for close at 1731847557484Writing region close event to WAL at 1731847557485 (+1 ms)Closed at 1731847557485 2024-11-17T12:45:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741920_1098 (size=175) 2024-11-17T12:45:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741920_1098 (size=175) 2024-11-17T12:45:57,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741920_1098 (size=175) 2024-11-17T12:45:57,489 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:45:57,489 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-75384987:(num 1731847557425) 2024-11-17T12:45:57,489 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:45:57,491 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:45:57,502 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491, exclude list is [], retry=0 2024-11-17T12:45:57,505 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:45:57,505 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:45:57,505 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:45:57,528 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 2024-11-17T12:45:57,528 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965)] 2024-11-17T12:45:57,528 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c92256f72a75329f061423749b98ffaf, NAME => 'testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:45:57,528 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:45:57,529 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,529 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,530 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,531 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName a 2024-11-17T12:45:57,531 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,531 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,531 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,532 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName b 2024-11-17T12:45:57,532 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,532 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,532 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,533 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName c 2024-11-17T12:45:57,533 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:45:57,533 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:45:57,533 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,534 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,535 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,536 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,536 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,536 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:45:57,538 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c92256f72a75329f061423749b98ffaf 2024-11-17T12:45:57,538 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c92256f72a75329f061423749b98ffaf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69744804, jitterRate=0.03927856683731079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:45:57,539 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c92256f72a75329f061423749b98ffaf: Writing region info on filesystem at 1731847557529Initializing all the Stores at 1731847557529Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557530 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557530Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847557530Cleaning up temporary data from old regions at 1731847557536 (+6 ms)Region opened successfully at 1731847557539 (+3 ms) 2024-11-17T12:45:57,542 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731847557541/Put/seqid=0 2024-11-17T12:45:57,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741923_1101 (size=4875) 2024-11-17T12:45:57,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741923_1101 (size=4875) 2024-11-17T12:45:57,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741923_1101 (size=4875) 2024-11-17T12:45:57,552 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731847557552/Put/seqid=0 2024-11-17T12:45:57,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741924_1102 (size=4875) 2024-11-17T12:45:57,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741924_1102 (size=4875) 2024-11-17T12:45:57,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741924_1102 (size=4875) 2024-11-17T12:45:57,567 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731847557567/Put/seqid=0 2024-11-17T12:45:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741925_1103 (size=4875) 2024-11-17T12:45:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741925_1103 (size=4875) 2024-11-17T12:45:57,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741925_1103 (size=4875) 2024-11-17T12:45:57,575 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in c92256f72a75329f061423749b98ffaf/a 2024-11-17T12:45:57,578 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-17T12:45:57,578 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-17T12:45:57,578 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in c92256f72a75329f061423749b98ffaf/a 2024-11-17T12:45:57,582 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-17T12:45:57,582 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-17T12:45:57,582 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in c92256f72a75329f061423749b98ffaf/a 2024-11-17T12:45:57,585 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-17T12:45:57,585 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-17T12:45:57,585 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c92256f72a75329f061423749b98ffaf 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-17T12:45:57,598 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/.tmp/a/1687839852dd46688c709638daaa1c5c is 55, key is testCompactedBulkLoadedFiles/a:a/1731847557539/Put/seqid=0 2024-11-17T12:45:57,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741926_1104 (size=5107) 2024-11-17T12:45:57,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741926_1104 (size=5107) 2024-11-17T12:45:57,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741926_1104 (size=5107) 2024-11-17T12:45:57,604 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/.tmp/a/1687839852dd46688c709638daaa1c5c 2024-11-17T12:45:57,609 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/.tmp/a/1687839852dd46688c709638daaa1c5c as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c 2024-11-17T12:45:57,616 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c, entries=1, sequenceid=4, filesize=5.0 K 2024-11-17T12:45:57,618 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for c92256f72a75329f061423749b98ffaf in 33ms, sequenceid=4, compaction requested=false 2024-11-17T12:45:57,618 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c92256f72a75329f061423749b98ffaf: 2024-11-17T12:45:57,620 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_ 2024-11-17T12:45:57,621 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_ 2024-11-17T12:45:57,622 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_ 2024-11-17T12:45:57,623 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile0 into c92256f72a75329f061423749b98ffaf/a as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_ - updating store file list. 2024-11-17T12:45:57,628 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-17T12:45:57,628 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_ into c92256f72a75329f061423749b98ffaf/a 2024-11-17T12:45:57,628 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile0 into c92256f72a75329f061423749b98ffaf/a (new location: hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_) 2024-11-17T12:45:57,630 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile1 into c92256f72a75329f061423749b98ffaf/a as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_ - updating store file list. 2024-11-17T12:45:57,635 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6ec8d51b2f6844058a399eda82da215f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-17T12:45:57,635 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_ into c92256f72a75329f061423749b98ffaf/a 2024-11-17T12:45:57,635 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile1 into c92256f72a75329f061423749b98ffaf/a (new location: hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_) 2024-11-17T12:45:57,636 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile2 into c92256f72a75329f061423749b98ffaf/a as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_ - updating store file list. 2024-11-17T12:45:57,640 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 10697683061e46e999848adddf82c87c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-17T12:45:57,640 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_ into c92256f72a75329f061423749b98ffaf/a 2024-11-17T12:45:57,640 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37973/hbase/testCompactedBulkLoadedFiles/hfile2 into c92256f72a75329f061423749b98ffaf/a (new location: hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_) 2024-11-17T12:45:57,646 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T12:45:57,647 DEBUG [Time-limited test {}] regionserver.HStore(1541): c92256f72a75329f061423749b98ffaf/a is initiating major compaction (all files) 2024-11-17T12:45:57,647 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of c92256f72a75329f061423749b98ffaf/a in testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:45:57,647 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_] into tmpdir=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/.tmp, totalSize=19.3 K 2024-11-17T12:45:57,647 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1687839852dd46688c709638daaa1c5c, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731847557539 2024-11-17T12:45:57,648 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-17T12:45:57,648 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6ec8d51b2f6844058a399eda82da215f_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-17T12:45:57,649 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 10697683061e46e999848adddf82c87c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-17T12:45:57,661 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/.tmp/a/aa933719a9784074aaf8d5da7a89e287 is 55, key is testCompactedBulkLoadedFiles/a:a/1731847557539/Put/seqid=0 2024-11-17T12:45:57,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741927_1105 (size=6154) 2024-11-17T12:45:57,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741927_1105 (size=6154) 2024-11-17T12:45:57,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741927_1105 (size=6154) 2024-11-17T12:45:57,674 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/.tmp/a/aa933719a9784074aaf8d5da7a89e287 as hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/aa933719a9784074aaf8d5da7a89e287 2024-11-17T12:45:57,681 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in c92256f72a75329f061423749b98ffaf/a of c92256f72a75329f061423749b98ffaf into aa933719a9784074aaf8d5da7a89e287(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T12:45:57,681 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for c92256f72a75329f061423749b98ffaf: 2024-11-17T12:45:57,681 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-17T12:45:57,681 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-17T12:45:57,710 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491, size=0 (0bytes) 2024-11-17T12:45:57,711 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 might be still open, length is 0 2024-11-17T12:45:57,711 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 2024-11-17T12:45:57,711 WARN [IPC Server handler 0 on default port 37973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-17T12:45:57,711 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 after 0ms 2024-11-17T12:46:00,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:41154 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34421:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41154 dst: /127.0.0.1:34421 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34421 remote=/127.0.0.1:41154]. Total timeout mills is 60000, 57403 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:00,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:34462 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:42393:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34462 dst: /127.0.0.1:42393 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:00,278 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:52450 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:44483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52450 dst: /127.0.0.1:44483 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:00,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741922_1106 (size=1794) 2024-11-17T12:46:00,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741922_1106 (size=1794) 2024-11-17T12:46:00,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741922_1106 (size=1794) 2024-11-17T12:46:01,712 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 after 4001ms 2024-11-17T12:46:01,737 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:46:01,738 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 took 4028ms 2024-11-17T12:46:01,740 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491; continuing. 2024-11-17T12:46:01,740 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 so closing down 2024-11-17T12:46:01,740 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:46:01,741 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731847557491.temp 2024-11-17T12:46:01,763 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000003-wal.1731847557491.temp 2024-11-17T12:46:01,764 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:46:01,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741928_1107 (size=800) 2024-11-17T12:46:01,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741928_1107 (size=800) 2024-11-17T12:46:01,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741928_1107 (size=800) 2024-11-17T12:46:01,770 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000003-wal.1731847557491.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-17T12:46:01,771 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000003-wal.1731847557491.temp to hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000008 2024-11-17T12:46:01,771 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491, size=0, length=0, corrupted=false, cancelled=false 2024-11-17T12:46:01,771 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491, journal: Splitting hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491, size=0 (0bytes) at 1731847557711Finishing writing output for hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 so closing down at 1731847561740 (+4029 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000003-wal.1731847557491.temp at 1731847561764 (+24 ms)3 split writer threads finished at 1731847561764Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000003-wal.1731847557491.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731847561770 (+6 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000003-wal.1731847557491.temp to hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000008 at 1731847561771 (+1 ms)Processed 5 edits across 1 Regions in 33 ms; skipped=3; WAL=hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491, size=0, length=0, corrupted=false, cancelled=false at 1731847561771 2024-11-17T12:46:01,773 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847557491 2024-11-17T12:46:01,774 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000008 2024-11-17T12:46:01,774 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:46:01,775 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:46:01,787 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847561776, exclude list is [], retry=0 2024-11-17T12:46:01,790 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:46:01,790 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:46:01,791 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:46:01,813 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847561776 2024-11-17T12:46:01,814 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:46:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => c92256f72a75329f061423749b98ffaf, NAME => 'testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:46:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:46:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,815 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,816 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName a 2024-11-17T12:46:01,816 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:01,822 DEBUG [StoreFileOpener-c92256f72a75329f061423749b98ffaf-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 10697683061e46e999848adddf82c87c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-17T12:46:01,823 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_ 2024-11-17T12:46:01,826 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c 2024-11-17T12:46:01,830 DEBUG [StoreFileOpener-c92256f72a75329f061423749b98ffaf-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6ec8d51b2f6844058a399eda82da215f_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-17T12:46:01,830 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_ 2024-11-17T12:46:01,834 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/aa933719a9784074aaf8d5da7a89e287 2024-11-17T12:46:01,838 DEBUG [StoreFileOpener-c92256f72a75329f061423749b98ffaf-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-17T12:46:01,838 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_ 2024-11-17T12:46:01,838 WARN [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@68c056a2 2024-11-17T12:46:01,838 WARN [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@68c056a2 2024-11-17T12:46:01,838 WARN [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@68c056a2 2024-11-17T12:46:01,838 WARN [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@68c056a2 2024-11-17T12:46:01,838 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_] to archive 2024-11-17T12:46:01,839 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T12:46:01,841 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_ to hdfs://localhost:37973/hbase/archive/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/10697683061e46e999848adddf82c87c_SeqId_4_ 2024-11-17T12:46:01,842 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c to hdfs://localhost:37973/hbase/archive/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/1687839852dd46688c709638daaa1c5c 2024-11-17T12:46:01,843 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_ to hdfs://localhost:37973/hbase/archive/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/6ec8d51b2f6844058a399eda82da215f_SeqId_4_ 2024-11-17T12:46:01,844 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_ to hdfs://localhost:37973/hbase/archive/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/a/cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_ 2024-11-17T12:46:01,844 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:01,845 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,845 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName b 2024-11-17T12:46:01,845 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:01,846 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:01,846 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,847 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c92256f72a75329f061423749b98ffaf columnFamilyName c 2024-11-17T12:46:01,847 DEBUG [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:01,847 INFO [StoreOpener-c92256f72a75329f061423749b98ffaf-1 {}] regionserver.HStore(327): Store=c92256f72a75329f061423749b98ffaf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:01,847 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,848 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,849 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,849 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000008 2024-11-17T12:46:01,861 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T12:46:01,874 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:46:01,876 DEBUG [Time-limited test {}] regionserver.HRegion(5836): c92256f72a75329f061423749b98ffaf : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "c92256f72a75329f061423749b98ffaf" family_name: "a" compaction_input: "1687839852dd46688c709638daaa1c5c" compaction_input: "cb8947f3d5aa40899e6db2d62eedc54a_SeqId_4_" compaction_input: "6ec8d51b2f6844058a399eda82da215f_SeqId_4_" compaction_input: "10697683061e46e999848adddf82c87c_SeqId_4_" compaction_output: "aa933719a9784074aaf8d5da7a89e287" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-17T12:46:01,876 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-17T12:46:01,876 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000008 2024-11-17T12:46:01,877 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/0000000000000000008 2024-11-17T12:46:01,878 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,878 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,879 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:46:01,880 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for c92256f72a75329f061423749b98ffaf 2024-11-17T12:46:01,883 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testCompactedBulkLoadedFiles/c92256f72a75329f061423749b98ffaf/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-17T12:46:01,884 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened c92256f72a75329f061423749b98ffaf; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73915483, jitterRate=0.10142652690410614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:46:01,884 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for c92256f72a75329f061423749b98ffaf: Writing region info on filesystem at 1731847561814Initializing all the Stores at 1731847561815 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847561815Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847561815Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847561815Cleaning up temporary data from old regions at 1731847561878 (+63 ms)Region opened successfully at 1731847561884 (+6 ms) 2024-11-17T12:46:01,886 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing c92256f72a75329f061423749b98ffaf, disabling compactions & flushes 2024-11-17T12:46:01,886 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:46:01,886 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:46:01,886 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. after waiting 0 ms 2024-11-17T12:46:01,886 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:46:01,887 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731847557421.c92256f72a75329f061423749b98ffaf. 2024-11-17T12:46:01,887 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for c92256f72a75329f061423749b98ffaf: Waiting for close lock at 1731847561886Disabling compacts and flushes for region at 1731847561886Disabling writes for close at 1731847561886Writing region close event to WAL at 1731847561887 (+1 ms)Closed at 1731847561887 2024-11-17T12:46:01,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741929_1108 (size=175) 2024-11-17T12:46:01,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741929_1108 (size=175) 2024-11-17T12:46:01,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741929_1108 (size=175) 2024-11-17T12:46:01,892 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:46:01,892 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731847561776) 2024-11-17T12:46:01,906 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=439 (was 425) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2019370793_22 at /127.0.0.1:34544 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:37973 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2019370793_22 at /127.0.0.1:52498 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:37973 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2019370793_22 at /127.0.0.1:41206 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=187 (was 195), ProcessCount=11 (was 11), AvailableMemoryMB=6044 (was 6055) 2024-11-17T12:46:01,906 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-17T12:46:01,917 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=439, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=187, ProcessCount=11, AvailableMemoryMB=6043 2024-11-17T12:46:01,917 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-17T12:46:01,930 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:46:01,932 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T12:46:01,932 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T12:46:01,935 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45280302, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/hregion-45280302, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:46:01,949 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45280302/hregion-45280302.1731847561935, exclude list is [], retry=0 2024-11-17T12:46:01,952 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:46:01,952 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:46:01,952 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:46:01,981 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45280302/hregion-45280302.1731847561935 2024-11-17T12:46:01,981 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:46:01,981 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => eab288560798b825d63770614391042c, NAME => 'testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37973/hbase 2024-11-17T12:46:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741931_1110 (size=67) 2024-11-17T12:46:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741931_1110 (size=67) 2024-11-17T12:46:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741931_1110 (size=67) 2024-11-17T12:46:01,990 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:46:01,992 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eab288560798b825d63770614391042c 2024-11-17T12:46:01,993 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName a 2024-11-17T12:46:01,993 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:01,994 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:01,994 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eab288560798b825d63770614391042c 2024-11-17T12:46:01,995 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName b 2024-11-17T12:46:01,995 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:01,995 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:01,996 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eab288560798b825d63770614391042c 2024-11-17T12:46:01,997 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName c 2024-11-17T12:46:01,997 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:01,997 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:01,997 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eab288560798b825d63770614391042c 2024-11-17T12:46:01,998 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:01,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:02,000 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eab288560798b825d63770614391042c 2024-11-17T12:46:02,000 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eab288560798b825d63770614391042c 2024-11-17T12:46:02,000 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:46:02,001 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eab288560798b825d63770614391042c 2024-11-17T12:46:02,004 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T12:46:02,004 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eab288560798b825d63770614391042c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60081616, jitterRate=-0.10471415519714355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:46:02,004 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eab288560798b825d63770614391042c: Writing region info on filesystem at 1731847561991Initializing all the Stores at 1731847561991Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847561991Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847561992 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847561992Cleaning up temporary data from old regions at 1731847562000 (+8 ms)Region opened successfully at 1731847562004 (+4 ms) 2024-11-17T12:46:02,004 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eab288560798b825d63770614391042c, disabling compactions & flushes 2024-11-17T12:46:02,004 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,004 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,004 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. after waiting 0 ms 2024-11-17T12:46:02,004 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,005 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,005 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eab288560798b825d63770614391042c: Waiting for close lock at 1731847562004Disabling compacts and flushes for region at 1731847562004Disabling writes for close at 1731847562004Writing region close event to WAL at 1731847562005 (+1 ms)Closed at 1731847562005 2024-11-17T12:46:02,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741930_1109 (size=175) 2024-11-17T12:46:02,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741930_1109 (size=175) 2024-11-17T12:46:02,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741930_1109 (size=175) 2024-11-17T12:46:02,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:46:02,009 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45280302:(num 1731847561935) 2024-11-17T12:46:02,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:46:02,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:46:02,024 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012, exclude list is [], retry=0 2024-11-17T12:46:02,026 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:46:02,026 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:46:02,027 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:46:02,051 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012 2024-11-17T12:46:02,052 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:46:02,052 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => eab288560798b825d63770614391042c, NAME => 'testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:46:02,052 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:46:02,052 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for eab288560798b825d63770614391042c 2024-11-17T12:46:02,052 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for eab288560798b825d63770614391042c 2024-11-17T12:46:02,053 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eab288560798b825d63770614391042c 2024-11-17T12:46:02,054 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName a 2024-11-17T12:46:02,055 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:02,055 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:02,055 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eab288560798b825d63770614391042c 2024-11-17T12:46:02,056 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName b 2024-11-17T12:46:02,056 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:02,057 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:02,057 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eab288560798b825d63770614391042c 2024-11-17T12:46:02,058 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName c 2024-11-17T12:46:02,058 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:02,058 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:02,058 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eab288560798b825d63770614391042c 2024-11-17T12:46:02,059 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:02,060 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:02,061 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eab288560798b825d63770614391042c 2024-11-17T12:46:02,061 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eab288560798b825d63770614391042c 2024-11-17T12:46:02,061 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:46:02,063 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eab288560798b825d63770614391042c 2024-11-17T12:46:02,063 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eab288560798b825d63770614391042c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64538899, jitterRate=-0.03829546272754669}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:46:02,063 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eab288560798b825d63770614391042c: Writing region info on filesystem at 1731847562052Initializing all the Stores at 1731847562053 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847562053Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847562053Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847562053Cleaning up temporary data from old regions at 1731847562061 (+8 ms)Region opened successfully at 1731847562063 (+2 ms) 2024-11-17T12:46:02,071 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eab288560798b825d63770614391042c 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-17T12:46:02,085 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/a/0181c3a7a7864425b5b8a81fa09b2201 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731847562064/Put/seqid=0 2024-11-17T12:46:02,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741933_1112 (size=5958) 2024-11-17T12:46:02,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741933_1112 (size=5958) 2024-11-17T12:46:02,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741933_1112 (size=5958) 2024-11-17T12:46:02,092 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/a/0181c3a7a7864425b5b8a81fa09b2201 2024-11-17T12:46:02,097 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/a/0181c3a7a7864425b5b8a81fa09b2201 as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/a/0181c3a7a7864425b5b8a81fa09b2201 2024-11-17T12:46:02,102 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/a/0181c3a7a7864425b5b8a81fa09b2201, entries=10, sequenceid=13, filesize=5.8 K 2024-11-17T12:46:02,103 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for eab288560798b825d63770614391042c in 32ms, sequenceid=13, compaction requested=false 2024-11-17T12:46:02,104 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eab288560798b825d63770614391042c: 2024-11-17T12:46:02,118 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eab288560798b825d63770614391042c, disabling compactions & flushes 2024-11-17T12:46:02,118 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,118 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,118 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. after waiting 0 ms 2024-11-17T12:46:02,118 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,119 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,119 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:02,119 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eab288560798b825d63770614391042c: Waiting for close lock at 1731847562118Disabling compacts and flushes for region at 1731847562118Disabling writes for close at 1731847562118Writing region close event to WAL at 1731847562119 (+1 ms)Closed at 1731847562119 2024-11-17T12:46:02,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741932_1111 (size=6243) 2024-11-17T12:46:02,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741932_1111 (size=6243) 2024-11-17T12:46:02,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741932_1111 (size=6243) 2024-11-17T12:46:02,139 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012, size=6.1 K (6243bytes) 2024-11-17T12:46:02,139 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012 2024-11-17T12:46:02,139 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012 after 0ms 2024-11-17T12:46:02,175 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:46:02,175 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012 took 36ms 2024-11-17T12:46:02,178 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012 so closing down 2024-11-17T12:46:02,178 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:46:02,179 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731847562012.temp 2024-11-17T12:46:02,207 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000003-wal.1731847562012.temp 2024-11-17T12:46:02,207 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:46:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741934_1113 (size=5575) 2024-11-17T12:46:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741934_1113 (size=5575) 2024-11-17T12:46:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741934_1113 (size=5575) 2024-11-17T12:46:02,214 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000003-wal.1731847562012.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-17T12:46:02,216 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000003-wal.1731847562012.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000035 2024-11-17T12:46:02,216 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-17T12:46:02,216 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012, journal: Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012, size=6.1 K (6243bytes) at 1731847562139Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012 so closing down at 1731847562178 (+39 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000003-wal.1731847562012.temp at 1731847562207 (+29 ms)3 split writer threads finished at 1731847562207Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000003-wal.1731847562012.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731847562214 (+7 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000003-wal.1731847562012.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000035 at 1731847562216 (+2 ms)Processed 32 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731847562216 2024-11-17T12:46:02,218 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562012 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847562012 2024-11-17T12:46:02,218 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000035 2024-11-17T12:46:02,219 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:46:02,220 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:46:02,232 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221, exclude list is [], retry=0 2024-11-17T12:46:02,234 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:46:02,235 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:46:02,235 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:46:02,259 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 2024-11-17T12:46:02,259 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38819:38819),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:33427:33427)] 2024-11-17T12:46:02,259 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => eab288560798b825d63770614391042c, NAME => 'testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c.', STARTKEY => '', ENDKEY => ''} 2024-11-17T12:46:02,260 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:46:02,260 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for eab288560798b825d63770614391042c 2024-11-17T12:46:02,260 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for eab288560798b825d63770614391042c 2024-11-17T12:46:02,261 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eab288560798b825d63770614391042c 2024-11-17T12:46:02,262 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName a 2024-11-17T12:46:02,262 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:02,269 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/a/0181c3a7a7864425b5b8a81fa09b2201 2024-11-17T12:46:02,269 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:02,269 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eab288560798b825d63770614391042c 2024-11-17T12:46:02,270 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName b 2024-11-17T12:46:02,270 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:02,271 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:02,271 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eab288560798b825d63770614391042c 2024-11-17T12:46:02,272 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName c 2024-11-17T12:46:02,272 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:02,272 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:02,273 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eab288560798b825d63770614391042c 2024-11-17T12:46:02,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:02,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:02,275 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000035 2024-11-17T12:46:02,300 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:46:02,301 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000035 2024-11-17T12:46:02,301 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eab288560798b825d63770614391042c 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-17T12:46:02,315 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/b/01cf2b74618a4c2286e55375027a672c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731847562104/Put/seqid=0 2024-11-17T12:46:02,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741936_1115 (size=5958) 2024-11-17T12:46:02,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741936_1115 (size=5958) 2024-11-17T12:46:02,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741936_1115 (size=5958) 2024-11-17T12:46:02,322 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/b/01cf2b74618a4c2286e55375027a672c 2024-11-17T12:46:02,342 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/c/56408bd8b2b549bba1338db36bc6fd3b is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731847562110/Put/seqid=0 2024-11-17T12:46:02,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741937_1116 (size=5958) 2024-11-17T12:46:02,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741937_1116 (size=5958) 2024-11-17T12:46:02,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741937_1116 (size=5958) 2024-11-17T12:46:02,348 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/c/56408bd8b2b549bba1338db36bc6fd3b 2024-11-17T12:46:02,355 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/b/01cf2b74618a4c2286e55375027a672c as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/b/01cf2b74618a4c2286e55375027a672c 2024-11-17T12:46:02,361 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/b/01cf2b74618a4c2286e55375027a672c, entries=10, sequenceid=35, filesize=5.8 K 2024-11-17T12:46:02,363 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/c/56408bd8b2b549bba1338db36bc6fd3b as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/c/56408bd8b2b549bba1338db36bc6fd3b 2024-11-17T12:46:02,368 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/c/56408bd8b2b549bba1338db36bc6fd3b, entries=10, sequenceid=35, filesize=5.8 K 2024-11-17T12:46:02,368 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for eab288560798b825d63770614391042c in 67ms, sequenceid=35, compaction requested=false; wal=null 2024-11-17T12:46:02,369 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000035 2024-11-17T12:46:02,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eab288560798b825d63770614391042c 2024-11-17T12:46:02,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eab288560798b825d63770614391042c 2024-11-17T12:46:02,371 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:46:02,373 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eab288560798b825d63770614391042c 2024-11-17T12:46:02,375 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-17T12:46:02,376 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eab288560798b825d63770614391042c; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72359826, jitterRate=0.07824543118476868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:46:02,377 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eab288560798b825d63770614391042c: Writing region info on filesystem at 1731847562260Initializing all the Stores at 1731847562261 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847562261Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847562261Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847562261Obtaining lock to block concurrent updates at 1731847562301 (+40 ms)Preparing flush snapshotting stores in eab288560798b825d63770614391042c at 1731847562301Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731847562301Flushing stores of testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. at 1731847562301Flushing eab288560798b825d63770614391042c/b: creating writer at 1731847562301Flushing eab288560798b825d63770614391042c/b: appending metadata at 1731847562314 (+13 ms)Flushing eab288560798b825d63770614391042c/b: closing flushed file at 1731847562314Flushing eab288560798b825d63770614391042c/c: creating writer at 1731847562327 (+13 ms)Flushing eab288560798b825d63770614391042c/c: appending metadata at 1731847562341 (+14 ms)Flushing eab288560798b825d63770614391042c/c: closing flushed file at 1731847562341Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77f8a8ab: reopening flushed file at 1731847562354 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@706a254b: reopening flushed file at 1731847562362 (+8 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for eab288560798b825d63770614391042c in 67ms, sequenceid=35, compaction requested=false; wal=null at 1731847562368 (+6 ms)Cleaning up temporary data from old regions at 1731847562370 (+2 ms)Region opened successfully at 1731847562377 (+7 ms) 2024-11-17T12:46:02,444 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221, size=0 (0bytes) 2024-11-17T12:46:02,444 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 might be still open, length is 0 2024-11-17T12:46:02,444 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 2024-11-17T12:46:02,444 WARN [IPC Server handler 3 on default port 37973 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-17T12:46:02,445 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 after 1ms 2024-11-17T12:46:03,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:38166 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34421:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38166 dst: /127.0.0.1:34421 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34421 remote=/127.0.0.1:38166]. Total timeout mills is 60000, 59133 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:03,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:48874 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:44483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48874 dst: /127.0.0.1:44483 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:03,275 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-781036887_22 at /127.0.0.1:55456 [Receiving block BP-131233367-172.17.0.2-1731847521320:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:42393:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55456 dst: /127.0.0.1:42393 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:03,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741935_1117 (size=5567) 2024-11-17T12:46:03,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741935_1117 (size=5567) 2024-11-17T12:46:05,568 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-17T12:46:05,568 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-17T12:46:05,570 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-17T12:46:05,570 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-17T12:46:06,446 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 after 4002ms 2024-11-17T12:46:06,479 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:46:06,479 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 took 4035ms 2024-11-17T12:46:06,481 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221; continuing. 2024-11-17T12:46:06,481 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 so closing down 2024-11-17T12:46:06,481 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-17T12:46:06,483 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731847562221.temp 2024-11-17T12:46:06,505 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000037-wal.1731847562221.temp 2024-11-17T12:46:06,505 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-17T12:46:06,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741938_1118 (size=5575) 2024-11-17T12:46:06,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741938_1118 (size=5575) 2024-11-17T12:46:06,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741938_1118 (size=5575) 2024-11-17T12:46:06,511 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000037-wal.1731847562221.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-17T12:46:06,512 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000037-wal.1731847562221.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000066 2024-11-17T12:46:06,512 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221, size=0, length=0, corrupted=false, cancelled=false 2024-11-17T12:46:06,513 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221, journal: Splitting hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221, size=0 (0bytes) at 1731847562444Finishing writing output for hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 so closing down at 1731847566481 (+4037 ms)Creating recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000037-wal.1731847562221.temp at 1731847566505 (+24 ms)3 split writer threads finished at 1731847566505Closed recovered edits writer path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000037-wal.1731847562221.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731847566511 (+6 ms)Rename recovered edits hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000037-wal.1731847562221.temp to hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000066 at 1731847566512 (+1 ms)Processed 30 edits across 1 Regions in 32 ms; skipped=0; WAL=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221, size=0, length=0, corrupted=false, cancelled=false at 1731847566512 2024-11-17T12:46:06,514 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 to hdfs://localhost:37973/hbase/oldWALs/wal.1731847562221 2024-11-17T12:46:06,515 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000066 2024-11-17T12:46:06,515 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-17T12:46:06,516 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37973/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929, archiveDir=hdfs://localhost:37973/hbase/oldWALs, maxLogs=32 2024-11-17T12:46:06,528 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847566517, exclude list is [], retry=0 2024-11-17T12:46:06,530 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42393,DS-b9ce6c0a-9af4-4909-8b20-c574197bc1df,DISK] 2024-11-17T12:46:06,530 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44483,DS-82575f11-d40e-49b1-b3c2-7b9c594a815d,DISK] 2024-11-17T12:46:06,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34421,DS-7f2e1da3-e7fa-45cb-9d82-b8545bcbe95c,DISK] 2024-11-17T12:46:06,554 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847566517 2024-11-17T12:46:06,554 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33427:33427),(127.0.0.1/127.0.0.1:42965:42965),(127.0.0.1/127.0.0.1:38819:38819)] 2024-11-17T12:46:06,554 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T12:46:06,556 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region eab288560798b825d63770614391042c 2024-11-17T12:46:06,556 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName a 2024-11-17T12:46:06,556 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:06,561 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/a/0181c3a7a7864425b5b8a81fa09b2201 2024-11-17T12:46:06,561 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:06,561 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region eab288560798b825d63770614391042c 2024-11-17T12:46:06,562 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName b 2024-11-17T12:46:06,562 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:06,567 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/b/01cf2b74618a4c2286e55375027a672c 2024-11-17T12:46:06,568 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:06,568 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region eab288560798b825d63770614391042c 2024-11-17T12:46:06,568 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eab288560798b825d63770614391042c columnFamilyName c 2024-11-17T12:46:06,569 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T12:46:06,574 DEBUG [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/c/56408bd8b2b549bba1338db36bc6fd3b 2024-11-17T12:46:06,574 INFO [StoreOpener-eab288560798b825d63770614391042c-1 {}] regionserver.HStore(327): Store=eab288560798b825d63770614391042c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T12:46:06,574 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for eab288560798b825d63770614391042c 2024-11-17T12:46:06,575 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:06,576 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c 2024-11-17T12:46:06,577 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000066 2024-11-17T12:46:06,601 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-17T12:46:06,604 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000066 2024-11-17T12:46:06,604 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eab288560798b825d63770614391042c 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-17T12:46:06,618 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/a/f917304b223c4afbaff7b5675b9bb22a is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731847562384/Put/seqid=0 2024-11-17T12:46:06,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741940_1120 (size=5958) 2024-11-17T12:46:06,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741940_1120 (size=5958) 2024-11-17T12:46:06,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741940_1120 (size=5958) 2024-11-17T12:46:06,624 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/a/f917304b223c4afbaff7b5675b9bb22a 2024-11-17T12:46:06,641 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/b/519e6013194f4ddfa68630274cacacd9 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731847562394/Put/seqid=0 2024-11-17T12:46:06,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741941_1121 (size=5958) 2024-11-17T12:46:06,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741941_1121 (size=5958) 2024-11-17T12:46:06,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741941_1121 (size=5958) 2024-11-17T12:46:06,648 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/b/519e6013194f4ddfa68630274cacacd9 2024-11-17T12:46:06,670 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/c/44cb4b043800488985b9437f92fbd92c is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731847562401/Put/seqid=0 2024-11-17T12:46:06,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741942_1122 (size=5958) 2024-11-17T12:46:06,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741942_1122 (size=5958) 2024-11-17T12:46:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741942_1122 (size=5958) 2024-11-17T12:46:06,676 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/c/44cb4b043800488985b9437f92fbd92c 2024-11-17T12:46:06,681 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/a/f917304b223c4afbaff7b5675b9bb22a as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/a/f917304b223c4afbaff7b5675b9bb22a 2024-11-17T12:46:06,687 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/a/f917304b223c4afbaff7b5675b9bb22a, entries=10, sequenceid=66, filesize=5.8 K 2024-11-17T12:46:06,688 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/b/519e6013194f4ddfa68630274cacacd9 as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/b/519e6013194f4ddfa68630274cacacd9 2024-11-17T12:46:06,692 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/b/519e6013194f4ddfa68630274cacacd9, entries=10, sequenceid=66, filesize=5.8 K 2024-11-17T12:46:06,693 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/.tmp/c/44cb4b043800488985b9437f92fbd92c as hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/c/44cb4b043800488985b9437f92fbd92c 2024-11-17T12:46:06,698 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/c/44cb4b043800488985b9437f92fbd92c, entries=10, sequenceid=66, filesize=5.8 K 2024-11-17T12:46:06,698 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for eab288560798b825d63770614391042c in 94ms, sequenceid=66, compaction requested=false; wal=null 2024-11-17T12:46:06,699 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/0000000000000000066 2024-11-17T12:46:06,700 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for eab288560798b825d63770614391042c 2024-11-17T12:46:06,700 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for eab288560798b825d63770614391042c 2024-11-17T12:46:06,701 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-17T12:46:06,702 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for eab288560798b825d63770614391042c 2024-11-17T12:46:06,704 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/hbase/data/default/testReplayEditsWrittenViaHRegion/eab288560798b825d63770614391042c/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-17T12:46:06,705 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened eab288560798b825d63770614391042c; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67124873, jitterRate=2.3855268955230713E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-17T12:46:06,705 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for eab288560798b825d63770614391042c: Writing region info on filesystem at 1731847566554Initializing all the Stores at 1731847566555 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847566555Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847566556 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731847566556Obtaining lock to block concurrent updates at 1731847566604 (+48 ms)Preparing flush snapshotting stores in eab288560798b825d63770614391042c at 1731847566604Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731847566605 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. at 1731847566605Flushing eab288560798b825d63770614391042c/a: creating writer at 1731847566605Flushing eab288560798b825d63770614391042c/a: appending metadata at 1731847566617 (+12 ms)Flushing eab288560798b825d63770614391042c/a: closing flushed file at 1731847566618 (+1 ms)Flushing eab288560798b825d63770614391042c/b: creating writer at 1731847566628 (+10 ms)Flushing eab288560798b825d63770614391042c/b: appending metadata at 1731847566641 (+13 ms)Flushing eab288560798b825d63770614391042c/b: closing flushed file at 1731847566641Flushing eab288560798b825d63770614391042c/c: creating writer at 1731847566653 (+12 ms)Flushing eab288560798b825d63770614391042c/c: appending metadata at 1731847566669 (+16 ms)Flushing eab288560798b825d63770614391042c/c: closing flushed file at 1731847566669Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22207c12: reopening flushed file at 1731847566680 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3831c9a6: reopening flushed file at 1731847566687 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ecf9c48: reopening flushed file at 1731847566692 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for eab288560798b825d63770614391042c in 94ms, sequenceid=66, compaction requested=false; wal=null at 1731847566698 (+6 ms)Cleaning up temporary data from old regions at 1731847566700 (+2 ms)Region opened successfully at 1731847566705 (+5 ms) 2024-11-17T12:46:06,717 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing eab288560798b825d63770614391042c, disabling compactions & flushes 2024-11-17T12:46:06,717 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:06,717 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:06,717 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. after waiting 0 ms 2024-11-17T12:46:06,717 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:06,718 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731847561931.eab288560798b825d63770614391042c. 2024-11-17T12:46:06,718 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for eab288560798b825d63770614391042c: Waiting for close lock at 1731847566717Disabling compacts and flushes for region at 1731847566717Disabling writes for close at 1731847566717Writing region close event to WAL at 1731847566718 (+1 ms)Closed at 1731847566718 2024-11-17T12:46:06,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741939_1119 (size=175) 2024-11-17T12:46:06,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741939_1119 (size=175) 2024-11-17T12:46:06,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741939_1119 (size=175) 2024-11-17T12:46:06,722 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-17T12:46:06,722 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731847566517) 2024-11-17T12:46:06,736 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=445 (was 439) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_868804204_22 at /127.0.0.1:55526 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_868804204_22 at /127.0.0.1:38194 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:37973 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_868804204_22 at /127.0.0.1:48930 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:37973 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=204 (was 187) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6023 (was 6043) 2024-11-17T12:46:06,736 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-17T12:46:06,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T12:46:06,736 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T12:46:06,737 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T12:46:06,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:46:06,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:46:06,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T12:46:06,737 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T12:46:06,737 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1516095396, stopped=false 2024-11-17T12:46:06,738 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0d751a622496,38549,1731847525223 2024-11-17T12:46:06,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T12:46:06,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T12:46:06,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T12:46:06,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:46:06,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:46:06,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:46:06,791 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T12:46:06,791 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T12:46:06,792 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T12:46:06,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:46:06,793 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,38707,1731847525964' ***** 2024-11-17T12:46:06,794 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T12:46:06,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T12:46:06,794 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0d751a622496,32849,1731847526132' ***** 2024-11-17T12:46:06,794 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T12:46:06,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T12:46:06,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T12:46:06,794 INFO [RS:2;0d751a622496:32849 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T12:46:06,794 INFO [RS:0;0d751a622496:38707 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T12:46:06,794 INFO [RS:2;0d751a622496:32849 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T12:46:06,795 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T12:46:06,795 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T12:46:06,795 INFO [RS:2;0d751a622496:32849 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T12:46:06,795 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(3091): Received CLOSE for 8e8bd106579494c8f12507d73483b546 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,38707,1731847525964 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0d751a622496:38707. 2024-11-17T12:46:06,795 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(959): stopping server 0d751a622496,32849,1731847526132 2024-11-17T12:46:06,795 DEBUG [RS:0;0d751a622496:38707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T12:46:06,795 INFO [RS:2;0d751a622496:32849 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T12:46:06,795 DEBUG [RS:0;0d751a622496:38707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:46:06,795 INFO [RS:2;0d751a622496:32849 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0d751a622496:32849. 2024-11-17T12:46:06,795 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8e8bd106579494c8f12507d73483b546, disabling compactions & flushes 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T12:46:06,795 DEBUG [RS:2;0d751a622496:32849 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T12:46:06,795 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:46:06,795 INFO [RS:0;0d751a622496:38707 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T12:46:06,796 DEBUG [RS:2;0d751a622496:32849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:46:06,796 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:46:06,796 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T12:46:06,796 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T12:46:06,796 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. after waiting 0 ms 2024-11-17T12:46:06,796 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1325): Online Regions={8e8bd106579494c8f12507d73483b546=testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546.} 2024-11-17T12:46:06,796 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:46:06,796 DEBUG [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1351): Waiting on 8e8bd106579494c8f12507d73483b546 2024-11-17T12:46:06,796 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T12:46:06,796 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T12:46:06,796 DEBUG [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T12:46:06,796 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T12:46:06,796 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T12:46:06,796 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T12:46:06,796 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T12:46:06,796 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T12:46:06,796 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-17T12:46:06,801 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8e8bd106579494c8f12507d73483b546/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-17T12:46:06,801 INFO [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:46:06,801 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8e8bd106579494c8f12507d73483b546: Waiting for close lock at 1731847566795Running coprocessor pre-close hooks at 1731847566795Disabling compacts and flushes for region at 1731847566795Disabling writes for close at 1731847566796 (+1 ms)Writing region close event to WAL at 1731847566797 (+1 ms)Running coprocessor post-close hooks at 1731847566801 (+4 ms)Closed at 1731847566801 2024-11-17T12:46:06,801 DEBUG [RS_CLOSE_REGION-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546. 2024-11-17T12:46:06,802 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/info/015990afbbd3443c9dc419bc6f5e3d5c is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731847541286.8e8bd106579494c8f12507d73483b546./info:regioninfo/1731847545377/Put/seqid=0 2024-11-17T12:46:06,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741943_1123 (size=6778) 2024-11-17T12:46:06,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741943_1123 (size=6778) 2024-11-17T12:46:06,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741943_1123 (size=6778) 2024-11-17T12:46:06,809 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/info/015990afbbd3443c9dc419bc6f5e3d5c 2024-11-17T12:46:06,814 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/.tmp/info/015990afbbd3443c9dc419bc6f5e3d5c as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/info/015990afbbd3443c9dc419bc6f5e3d5c 2024-11-17T12:46:06,818 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/info/015990afbbd3443c9dc419bc6f5e3d5c, entries=8, sequenceid=23, filesize=6.6 K 2024-11-17T12:46:06,819 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 23ms, sequenceid=23, compaction requested=false 2024-11-17T12:46:06,820 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T12:46:06,825 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-17T12:46:06,826 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T12:46:06,826 INFO [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T12:46:06,826 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731847566796Running coprocessor pre-close hooks at 1731847566796Disabling compacts and flushes for region at 1731847566796Disabling writes for close at 1731847566796Obtaining lock to block concurrent updates at 1731847566796Preparing flush snapshotting stores in 1588230740 at 1731847566796Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731847566797 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731847566798 (+1 ms)Flushing 1588230740/info: creating writer at 1731847566798Flushing 1588230740/info: appending metadata at 1731847566801 (+3 ms)Flushing 1588230740/info: closing flushed file at 1731847566801Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28405016: reopening flushed file at 1731847566813 (+12 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 23ms, sequenceid=23, compaction requested=false at 1731847566819 (+6 ms)Writing region close event to WAL at 1731847566821 (+2 ms)Running coprocessor post-close hooks at 1731847566826 (+5 ms)Closed at 1731847566826 2024-11-17T12:46:06,826 DEBUG [RS_CLOSE_META-regionserver/0d751a622496:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T12:46:06,882 INFO [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T12:46:06,882 INFO [regionserver/0d751a622496:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T12:46:06,996 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,32849,1731847526132; all regions closed. 2024-11-17T12:46:06,996 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(976): stopping server 0d751a622496,38707,1731847525964; all regions closed. 2024-11-17T12:46:07,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741893_1071 (size=2625) 2024-11-17T12:46:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741893_1071 (size=2625) 2024-11-17T12:46:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741893_1071 (size=2625) 2024-11-17T12:46:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741834_1010 (size=1032) 2024-11-17T12:46:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741834_1010 (size=1032) 2024-11-17T12:46:07,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741834_1010 (size=1032) 2024-11-17T12:46:07,007 DEBUG [RS:0;0d751a622496:38707 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs 2024-11-17T12:46:07,007 DEBUG [RS:2;0d751a622496:32849 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs 2024-11-17T12:46:07,007 INFO [RS:0;0d751a622496:38707 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0d751a622496%2C38707%2C1731847525964.meta:.meta(num 1731847544480) 2024-11-17T12:46:07,007 INFO [RS:2;0d751a622496:32849 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0d751a622496%2C32849%2C1731847526132:(num 1731847527698) 2024-11-17T12:46:07,007 DEBUG [RS:2;0d751a622496:32849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:46:07,007 INFO [RS:2;0d751a622496:32849 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T12:46:07,008 INFO [RS:2;0d751a622496:32849 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T12:46:07,008 INFO [RS:2;0d751a622496:32849 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T12:46:07,008 INFO [RS:2;0d751a622496:32849 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T12:46:07,008 INFO [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T12:46:07,008 INFO [RS:2;0d751a622496:32849 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T12:46:07,008 INFO [RS:2;0d751a622496:32849 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T12:46:07,008 INFO [RS:2;0d751a622496:32849 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T12:46:07,008 INFO [RS:2;0d751a622496:32849 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32849 2024-11-17T12:46:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741835_1011 (size=2764) 2024-11-17T12:46:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741835_1011 (size=2764) 2024-11-17T12:46:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741835_1011 (size=2764) 2024-11-17T12:46:07,012 DEBUG [RS:0;0d751a622496:38707 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/oldWALs 2024-11-17T12:46:07,012 INFO [RS:0;0d751a622496:38707 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0d751a622496%2C38707%2C1731847525964:(num 1731847527698) 2024-11-17T12:46:07,012 DEBUG [RS:0;0d751a622496:38707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T12:46:07,012 INFO [RS:0;0d751a622496:38707 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T12:46:07,012 INFO [RS:0;0d751a622496:38707 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T12:46:07,012 INFO [RS:0;0d751a622496:38707 {}] hbase.ChoreService(370): Chore service for: regionserver/0d751a622496:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T12:46:07,012 INFO [RS:0;0d751a622496:38707 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T12:46:07,012 INFO [regionserver/0d751a622496:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T12:46:07,012 INFO [RS:0;0d751a622496:38707 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38707 2024-11-17T12:46:07,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T12:46:07,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,32849,1731847526132 2024-11-17T12:46:07,035 INFO [RS:2;0d751a622496:32849 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T12:46:07,043 INFO [RS:0;0d751a622496:38707 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T12:46:07,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0d751a622496,38707,1731847525964 2024-11-17T12:46:07,051 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,32849,1731847526132] 2024-11-17T12:46:07,068 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,32849,1731847526132 already deleted, retry=false 2024-11-17T12:46:07,068 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,32849,1731847526132 expired; onlineServers=1 2024-11-17T12:46:07,068 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0d751a622496,38707,1731847525964] 2024-11-17T12:46:07,076 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0d751a622496,38707,1731847525964 already deleted, retry=false 2024-11-17T12:46:07,076 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0d751a622496,38707,1731847525964 expired; onlineServers=0 2024-11-17T12:46:07,076 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0d751a622496,38549,1731847525223' ***** 2024-11-17T12:46:07,076 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T12:46:07,076 INFO [M:0;0d751a622496:38549 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T12:46:07,076 INFO [M:0;0d751a622496:38549 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T12:46:07,077 DEBUG [M:0;0d751a622496:38549 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T12:46:07,077 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T12:46:07,077 DEBUG [M:0;0d751a622496:38549 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T12:46:07,077 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847527315 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.large.0-1731847527315,5,FailOnTimeoutGroup] 2024-11-17T12:46:07,077 DEBUG [master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847527316 {}] cleaner.HFileCleaner(306): Exit Thread[master/0d751a622496:0:becomeActiveMaster-HFileCleaner.small.0-1731847527316,5,FailOnTimeoutGroup] 2024-11-17T12:46:07,077 INFO [M:0;0d751a622496:38549 {}] hbase.ChoreService(370): Chore service for: master/0d751a622496:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T12:46:07,077 INFO [M:0;0d751a622496:38549 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T12:46:07,078 DEBUG [M:0;0d751a622496:38549 {}] master.HMaster(1795): Stopping service threads 2024-11-17T12:46:07,078 INFO [M:0;0d751a622496:38549 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T12:46:07,078 INFO [M:0;0d751a622496:38549 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T12:46:07,079 INFO [M:0;0d751a622496:38549 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T12:46:07,079 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T12:46:07,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T12:46:07,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T12:46:07,085 DEBUG [M:0;0d751a622496:38549 {}] zookeeper.ZKUtil(347): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T12:46:07,085 WARN [M:0;0d751a622496:38549 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T12:46:07,086 INFO [M:0;0d751a622496:38549 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/.lastflushedseqids 2024-11-17T12:46:07,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741944_1124 (size=119) 2024-11-17T12:46:07,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741944_1124 (size=119) 2024-11-17T12:46:07,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741944_1124 (size=119) 2024-11-17T12:46:07,100 INFO [M:0;0d751a622496:38549 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T12:46:07,100 INFO [M:0;0d751a622496:38549 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T12:46:07,100 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T12:46:07,100 INFO [M:0;0d751a622496:38549 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:46:07,100 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:46:07,100 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T12:46:07,100 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:46:07,100 INFO [M:0;0d751a622496:38549 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-11-17T12:46:07,115 DEBUG [M:0;0d751a622496:38549 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad46399f9cc048d4910fb3c485ba6368 is 82, key is hbase:meta,,1/info:regioninfo/1731847544674/Put/seqid=0 2024-11-17T12:46:07,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741945_1125 (size=6063) 2024-11-17T12:46:07,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741945_1125 (size=6063) 2024-11-17T12:46:07,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741945_1125 (size=6063) 2024-11-17T12:46:07,121 INFO [M:0;0d751a622496:38549 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad46399f9cc048d4910fb3c485ba6368 2024-11-17T12:46:07,140 DEBUG [M:0;0d751a622496:38549 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9ed05e6f99494420b44d22a00171abbe is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731847541709/Put/seqid=0 2024-11-17T12:46:07,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741946_1126 (size=7908) 2024-11-17T12:46:07,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741946_1126 (size=7908) 2024-11-17T12:46:07,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741946_1126 (size=7908) 2024-11-17T12:46:07,149 INFO [M:0;0d751a622496:38549 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9ed05e6f99494420b44d22a00171abbe 2024-11-17T12:46:07,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:46:07,151 INFO [RS:2;0d751a622496:32849 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T12:46:07,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32849-0x1014a1da31f0003, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:46:07,151 INFO [RS:2;0d751a622496:32849 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,32849,1731847526132; zookeeper connection closed. 2024-11-17T12:46:07,152 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b3a80c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b3a80c6 2024-11-17T12:46:07,153 INFO [M:0;0d751a622496:38549 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9ed05e6f99494420b44d22a00171abbe 2024-11-17T12:46:07,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:46:07,160 INFO [RS:0;0d751a622496:38707 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T12:46:07,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38707-0x1014a1da31f0001, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:46:07,160 INFO [RS:0;0d751a622496:38707 {}] regionserver.HRegionServer(1031): Exiting; stopping=0d751a622496,38707,1731847525964; zookeeper connection closed. 2024-11-17T12:46:07,160 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@71a178f5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@71a178f5 2024-11-17T12:46:07,160 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-17T12:46:07,166 DEBUG [M:0;0d751a622496:38549 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ed1853b7580a4261bf3f4bca5ed647e6 is 69, key is 0d751a622496,32849,1731847526132/rs:state/1731847527408/Put/seqid=0 2024-11-17T12:46:07,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741947_1127 (size=5440) 2024-11-17T12:46:07,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741947_1127 (size=5440) 2024-11-17T12:46:07,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741947_1127 (size=5440) 2024-11-17T12:46:07,171 INFO [M:0;0d751a622496:38549 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ed1853b7580a4261bf3f4bca5ed647e6 2024-11-17T12:46:07,175 INFO [M:0;0d751a622496:38549 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed1853b7580a4261bf3f4bca5ed647e6 2024-11-17T12:46:07,176 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ad46399f9cc048d4910fb3c485ba6368 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ad46399f9cc048d4910fb3c485ba6368 2024-11-17T12:46:07,180 INFO [M:0;0d751a622496:38549 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ad46399f9cc048d4910fb3c485ba6368, entries=14, sequenceid=207, filesize=5.9 K 2024-11-17T12:46:07,181 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9ed05e6f99494420b44d22a00171abbe as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9ed05e6f99494420b44d22a00171abbe 2024-11-17T12:46:07,186 INFO [M:0;0d751a622496:38549 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9ed05e6f99494420b44d22a00171abbe 2024-11-17T12:46:07,186 INFO [M:0;0d751a622496:38549 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9ed05e6f99494420b44d22a00171abbe, entries=21, sequenceid=207, filesize=7.7 K 2024-11-17T12:46:07,187 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ed1853b7580a4261bf3f4bca5ed647e6 as hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ed1853b7580a4261bf3f4bca5ed647e6 2024-11-17T12:46:07,191 INFO [M:0;0d751a622496:38549 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ed1853b7580a4261bf3f4bca5ed647e6 2024-11-17T12:46:07,191 INFO [M:0;0d751a622496:38549 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37973/user/jenkins/test-data/d6b8315e-175d-d195-e24e-07083c32a5b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ed1853b7580a4261bf3f4bca5ed647e6, entries=3, sequenceid=207, filesize=5.3 K 2024-11-17T12:46:07,193 INFO [M:0;0d751a622496:38549 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=207, compaction requested=false 2024-11-17T12:46:07,194 INFO [M:0;0d751a622496:38549 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T12:46:07,194 DEBUG [M:0;0d751a622496:38549 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731847567100Disabling compacts and flushes for region at 1731847567100Disabling writes for close at 1731847567100Obtaining lock to block concurrent updates at 1731847567100Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731847567100Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1731847567101 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731847567101Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731847567101Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731847567114 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731847567114Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731847567125 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731847567140 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731847567140Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731847567153 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731847567165 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731847567165Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54773de6: reopening flushed file at 1731847567175 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@384d0307: reopening flushed file at 1731847567181 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e4ef63a: reopening flushed file at 1731847567186 (+5 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=207, compaction requested=false at 1731847567193 (+7 ms)Writing region close event to WAL at 1731847567194 (+1 ms)Closed at 1731847567194 2024-11-17T12:46:07,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44483 is added to blk_1073741830_1006 (size=102174) 2024-11-17T12:46:07,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42393 is added to blk_1073741830_1006 (size=102174) 2024-11-17T12:46:07,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34421 is added to blk_1073741830_1006 (size=102174) 2024-11-17T12:46:07,197 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T12:46:07,198 INFO [M:0;0d751a622496:38549 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T12:46:07,198 INFO [M:0;0d751a622496:38549 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38549 2024-11-17T12:46:07,198 INFO [M:0;0d751a622496:38549 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T12:46:07,326 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:46:07,326 INFO [M:0;0d751a622496:38549 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T12:46:07,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38549-0x1014a1da31f0000, quorum=127.0.0.1:59814, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T12:46:07,338 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847556434 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847556434 (inode 16688) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731847547019/wal.1731847556434 (inode 16688) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-17T12:46:07,341 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546853 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546853 (inode 16665) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731847546076/wal.1731847546853 (inode 16665) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-17T12:46:07,342 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731847536776/wal.1731847536897 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:07,344 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847557235 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847557235 (inode 16714) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731847556654/wal.1731847557235 (inode 16714) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-17T12:46:07,345 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731847557420/wal.1731847557491 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:07,345 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731847561929/wal.1731847562221 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:07,345 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731847529843/wal.1731847529936 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T12:46:07,347 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal.1731847529674 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal.1731847529674 (inode 16485) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731847529303/wal.1731847529674 (inode 16485) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-17T12:46:07,349 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal.1731847529013 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal.1731847529013 (inode 16462) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731847528580/wal.1731847529013 (inode 16462) Holder DFSClient_NONMAPREDUCE_-781036887_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-17T12:46:07,353 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f0cddd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T12:46:07,355 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@25cdce3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T12:46:07,356 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T12:46:07,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1846121a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T12:46:07,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7161498b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,STOPPED} 2024-11-17T12:46:07,358 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T12:46:07,358 WARN [BP-131233367-172.17.0.2-1731847521320 heartbeating to localhost/127.0.0.1:37973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T12:46:07,358 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T12:46:07,358 WARN [BP-131233367-172.17.0.2-1731847521320 heartbeating to localhost/127.0.0.1:37973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-131233367-172.17.0.2-1731847521320 (Datanode Uuid 6fb5cbbe-7ba9-44a1-9898-0f10a7eae599) service to localhost/127.0.0.1:37973 2024-11-17T12:46:07,360 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data5/current/BP-131233367-172.17.0.2-1731847521320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T12:46:07,360 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data6/current/BP-131233367-172.17.0.2-1731847521320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T12:46:07,361 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T12:46:07,362 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44df5d9d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T12:46:07,363 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@56fbb88e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T12:46:07,363 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T12:46:07,363 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40a3a086{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T12:46:07,363 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b034666{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,STOPPED} 2024-11-17T12:46:07,364 WARN [BP-131233367-172.17.0.2-1731847521320 heartbeating to localhost/127.0.0.1:37973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T12:46:07,364 WARN [BP-131233367-172.17.0.2-1731847521320 heartbeating to localhost/127.0.0.1:37973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-131233367-172.17.0.2-1731847521320 (Datanode Uuid 9612c832-297d-4827-8928-65664340be6b) service to localhost/127.0.0.1:37973 2024-11-17T12:46:07,365 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data3/current/BP-131233367-172.17.0.2-1731847521320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T12:46:07,365 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data4/current/BP-131233367-172.17.0.2-1731847521320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T12:46:07,365 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T12:46:07,365 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T12:46:07,365 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T12:46:07,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b57af8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T12:46:07,368 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@461b5ea8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T12:46:07,368 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T12:46:07,368 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e9da5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T12:46:07,369 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cf706e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,STOPPED} 2024-11-17T12:46:07,370 WARN [BP-131233367-172.17.0.2-1731847521320 heartbeating to localhost/127.0.0.1:37973 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T12:46:07,370 WARN [BP-131233367-172.17.0.2-1731847521320 heartbeating to localhost/127.0.0.1:37973 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-131233367-172.17.0.2-1731847521320 (Datanode Uuid 3e61f543-d328-43aa-92e4-5687ae6794a4) service to localhost/127.0.0.1:37973 2024-11-17T12:46:07,370 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T12:46:07,370 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T12:46:07,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data1/current/BP-131233367-172.17.0.2-1731847521320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T12:46:07,371 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/cluster_98031011-4437-b609-7609-2aa5a834df1e/data/data2/current/BP-131233367-172.17.0.2-1731847521320 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T12:46:07,371 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T12:46:07,377 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5608f426{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T12:46:07,378 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T12:46:07,378 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T12:46:07,378 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T12:46:07,378 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5c11920a-d081-fb72-a9a0-f670aa6e58df/hadoop.log.dir/,STOPPED} 2024-11-17T12:46:07,388 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T12:46:07,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down