2024-12-04 09:44:27,100 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 09:44:27,116 main DEBUG Took 0.013153 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 09:44:27,117 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 09:44:27,117 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 09:44:27,118 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 09:44:27,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,127 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 09:44:27,141 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,143 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,144 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,144 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,145 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,146 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,146 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,147 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,147 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,148 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,148 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,149 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,149 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,150 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,150 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,151 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,151 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,152 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,152 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,153 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,153 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,154 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,154 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 09:44:27,155 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,155 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 09:44:27,157 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 09:44:27,159 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 09:44:27,161 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 09:44:27,161 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 09:44:27,163 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 09:44:27,163 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 09:44:27,174 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 09:44:27,177 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 09:44:27,179 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 09:44:27,180 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 09:44:27,180 main DEBUG createAppenders(={Console}) 2024-12-04 09:44:27,181 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-04 09:44:27,181 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 09:44:27,182 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-04 09:44:27,183 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 09:44:27,183 main DEBUG OutputStream closed 2024-12-04 09:44:27,183 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 09:44:27,184 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 09:44:27,184 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-04 09:44:27,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 09:44:27,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 09:44:27,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 09:44:27,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 09:44:27,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 09:44:27,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 09:44:27,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 09:44:27,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 09:44:27,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 09:44:27,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 09:44:27,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 09:44:27,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 09:44:27,256 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 09:44:27,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 09:44:27,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 09:44:27,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 09:44:27,257 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 09:44:27,258 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 09:44:27,260 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 09:44:27,261 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-04 09:44:27,261 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 09:44:27,262 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-04T09:44:27,472 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56 2024-12-04 09:44:27,475 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 09:44:27,476 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T09:44:27,483 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-04T09:44:27,490 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-04T09:44:27,511 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T09:44:27,551 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T09:44:27,552 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T09:44:27,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T09:44:27,575 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b, deleteOnExit=true 2024-12-04T09:44:27,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T09:44:27,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/test.cache.data in system properties and HBase conf 2024-12-04T09:44:27,576 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T09:44:27,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir in system properties and HBase conf 2024-12-04T09:44:27,577 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T09:44:27,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T09:44:27,578 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T09:44:27,659 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T09:44:27,750 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T09:44:27,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T09:44:27,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T09:44:27,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T09:44:27,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T09:44:27,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T09:44:27,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T09:44:27,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T09:44:27,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T09:44:27,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T09:44:27,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/nfs.dump.dir in system properties and HBase conf 2024-12-04T09:44:27,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/java.io.tmpdir in system properties and HBase conf 2024-12-04T09:44:27,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T09:44:27,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T09:44:27,762 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T09:44:28,658 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T09:44:28,722 INFO [Time-limited test {}] log.Log(170): Logging initialized @2273ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T09:44:28,784 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T09:44:28,843 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T09:44:28,862 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T09:44:28,863 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T09:44:28,864 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T09:44:28,874 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T09:44:28,876 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@77da9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,AVAILABLE} 2024-12-04T09:44:28,877 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43ac9415{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T09:44:29,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@12264dc8{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/java.io.tmpdir/jetty-localhost-39783-hadoop-hdfs-3_4_1-tests_jar-_-any-4442928143777370261/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T09:44:29,027 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ce294cb{HTTP/1.1, (http/1.1)}{localhost:39783} 2024-12-04T09:44:29,027 INFO [Time-limited test {}] server.Server(415): Started @2578ms 2024-12-04T09:44:29,472 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T09:44:29,480 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T09:44:29,481 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T09:44:29,482 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T09:44:29,482 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T09:44:29,482 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@481eb1b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,AVAILABLE} 2024-12-04T09:44:29,483 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@991988e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T09:44:29,588 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3a1c1498{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/java.io.tmpdir/jetty-localhost-42191-hadoop-hdfs-3_4_1-tests_jar-_-any-11414540050073645283/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T09:44:29,588 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3badc187{HTTP/1.1, (http/1.1)}{localhost:42191} 2024-12-04T09:44:29,588 INFO [Time-limited test {}] server.Server(415): Started @3140ms 2024-12-04T09:44:29,637 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T09:44:29,767 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T09:44:29,771 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T09:44:29,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T09:44:29,775 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T09:44:29,775 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T09:44:29,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@157ee20f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,AVAILABLE} 2024-12-04T09:44:29,776 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@424d969a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T09:44:29,872 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@41fe4ca9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/java.io.tmpdir/jetty-localhost-37815-hadoop-hdfs-3_4_1-tests_jar-_-any-17364797733588153780/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T09:44:29,873 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@74ee88d2{HTTP/1.1, (http/1.1)}{localhost:37815} 2024-12-04T09:44:29,873 INFO [Time-limited test {}] server.Server(415): Started @3424ms 2024-12-04T09:44:29,875 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T09:44:29,916 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T09:44:29,920 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T09:44:29,923 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T09:44:29,923 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T09:44:29,923 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T09:44:29,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d458b2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,AVAILABLE} 2024-12-04T09:44:29,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de3b13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T09:44:30,019 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@31a39bd8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/java.io.tmpdir/jetty-localhost-41149-hadoop-hdfs-3_4_1-tests_jar-_-any-16983947990340318300/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T09:44:30,020 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5aa63b11{HTTP/1.1, (http/1.1)}{localhost:41149} 2024-12-04T09:44:30,020 INFO [Time-limited test {}] server.Server(415): Started @3571ms 2024-12-04T09:44:30,022 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T09:44:30,747 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data3/current/BP-129416318-172.17.0.2-1733305468255/current, will proceed with Du for space computation calculation, 2024-12-04T09:44:30,747 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data2/current/BP-129416318-172.17.0.2-1733305468255/current, will proceed with Du for space computation calculation, 2024-12-04T09:44:30,747 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data4/current/BP-129416318-172.17.0.2-1733305468255/current, will proceed with Du for space computation calculation, 2024-12-04T09:44:30,747 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data1/current/BP-129416318-172.17.0.2-1733305468255/current, will proceed with Du for space computation calculation, 2024-12-04T09:44:30,774 WARN [Thread-135 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data5/current/BP-129416318-172.17.0.2-1733305468255/current, will proceed with Du for space computation calculation, 2024-12-04T09:44:30,774 WARN [Thread-137 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data6/current/BP-129416318-172.17.0.2-1733305468255/current, will proceed with Du for space computation calculation, 2024-12-04T09:44:30,780 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T09:44:30,780 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T09:44:30,810 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T09:44:30,827 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87fbdbdbb510884a with lease ID 0xf9359a2c7b1e3a55: Processing first storage report for DS-ee3ed8ce-078d-4e16-accd-040556b684ca from datanode DatanodeRegistration(127.0.0.1:36275, datanodeUuid=d8d59ee4-b57e-4493-9cb4-2891c68d537e, infoPort=43537, infoSecurePort=0, ipcPort=36281, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255) 2024-12-04T09:44:30,828 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87fbdbdbb510884a with lease ID 0xf9359a2c7b1e3a55: from storage DS-ee3ed8ce-078d-4e16-accd-040556b684ca node DatanodeRegistration(127.0.0.1:36275, datanodeUuid=d8d59ee4-b57e-4493-9cb4-2891c68d537e, infoPort=43537, infoSecurePort=0, ipcPort=36281, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T09:44:30,829 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x15afbab01a841196 with lease ID 0xf9359a2c7b1e3a54: Processing first storage report for DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0 from datanode DatanodeRegistration(127.0.0.1:33979, datanodeUuid=5db4c2f1-0af9-47e1-9734-52e71841f250, infoPort=35799, infoSecurePort=0, ipcPort=45719, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255) 2024-12-04T09:44:30,829 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x15afbab01a841196 with lease ID 0xf9359a2c7b1e3a54: from storage DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0 node DatanodeRegistration(127.0.0.1:33979, datanodeUuid=5db4c2f1-0af9-47e1-9734-52e71841f250, infoPort=35799, infoSecurePort=0, ipcPort=45719, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T09:44:30,829 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeadd401d3d6227e8 with lease ID 0xf9359a2c7b1e3a56: Processing first storage report for DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6 from datanode DatanodeRegistration(127.0.0.1:40851, datanodeUuid=784e3a01-d4c8-46ff-b691-fd45daadc952, infoPort=44681, infoSecurePort=0, ipcPort=36031, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255) 2024-12-04T09:44:30,829 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeadd401d3d6227e8 with lease ID 0xf9359a2c7b1e3a56: from storage DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6 node DatanodeRegistration(127.0.0.1:40851, datanodeUuid=784e3a01-d4c8-46ff-b691-fd45daadc952, infoPort=44681, infoSecurePort=0, ipcPort=36031, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T09:44:30,829 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87fbdbdbb510884a with lease ID 0xf9359a2c7b1e3a55: Processing first storage report for DS-5016c25d-effe-4bff-8696-95f4f7341278 from datanode DatanodeRegistration(127.0.0.1:36275, datanodeUuid=d8d59ee4-b57e-4493-9cb4-2891c68d537e, infoPort=43537, infoSecurePort=0, ipcPort=36281, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255) 2024-12-04T09:44:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87fbdbdbb510884a with lease ID 0xf9359a2c7b1e3a55: from storage DS-5016c25d-effe-4bff-8696-95f4f7341278 node DatanodeRegistration(127.0.0.1:36275, datanodeUuid=d8d59ee4-b57e-4493-9cb4-2891c68d537e, infoPort=43537, infoSecurePort=0, ipcPort=36281, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T09:44:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x15afbab01a841196 with lease ID 0xf9359a2c7b1e3a54: Processing first storage report for DS-534a48f7-08ee-442a-87e6-4de4aa6ad44a from datanode DatanodeRegistration(127.0.0.1:33979, datanodeUuid=5db4c2f1-0af9-47e1-9734-52e71841f250, infoPort=35799, infoSecurePort=0, ipcPort=45719, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255) 2024-12-04T09:44:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x15afbab01a841196 with lease ID 0xf9359a2c7b1e3a54: from storage DS-534a48f7-08ee-442a-87e6-4de4aa6ad44a node DatanodeRegistration(127.0.0.1:33979, datanodeUuid=5db4c2f1-0af9-47e1-9734-52e71841f250, infoPort=35799, infoSecurePort=0, ipcPort=45719, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T09:44:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeadd401d3d6227e8 with lease ID 0xf9359a2c7b1e3a56: Processing first storage report for DS-3f060eab-a146-4a76-afd7-e801e0ffb0ec from datanode DatanodeRegistration(127.0.0.1:40851, datanodeUuid=784e3a01-d4c8-46ff-b691-fd45daadc952, infoPort=44681, infoSecurePort=0, ipcPort=36031, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255) 2024-12-04T09:44:30,830 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeadd401d3d6227e8 with lease ID 0xf9359a2c7b1e3a56: from storage DS-3f060eab-a146-4a76-afd7-e801e0ffb0ec node DatanodeRegistration(127.0.0.1:40851, datanodeUuid=784e3a01-d4c8-46ff-b691-fd45daadc952, infoPort=44681, infoSecurePort=0, ipcPort=36031, storageInfo=lv=-57;cid=testClusterID;nsid=1474900157;c=1733305468255), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T09:44:30,929 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56 2024-12-04T09:44:31,006 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/zookeeper_0, clientPort=64397, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T09:44:31,014 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64397 2024-12-04T09:44:31,026 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:31,029 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:31,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741825_1001 (size=7) 2024-12-04T09:44:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741825_1001 (size=7) 2024-12-04T09:44:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741825_1001 (size=7) 2024-12-04T09:44:31,604 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095 with version=8 2024-12-04T09:44:31,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/hbase-staging 2024-12-04T09:44:31,863 INFO [Time-limited test {}] client.ConnectionUtils(128): master/84486a41f81c:0 server-side Connection retries=45 2024-12-04T09:44:31,871 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:31,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:31,876 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T09:44:31,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:31,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T09:44:32,022 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T09:44:32,079 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T09:44:32,088 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T09:44:32,091 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T09:44:32,111 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 69530 (auto-detected) 2024-12-04T09:44:32,112 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T09:44:32,130 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38097 2024-12-04T09:44:32,152 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38097 connecting to ZooKeeper ensemble=127.0.0.1:64397 2024-12-04T09:44:32,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:380970x0, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T09:44:32,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38097-0x101a10432830000 connected 2024-12-04T09:44:32,354 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,360 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T09:44:32,373 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095, hbase.cluster.distributed=false 2024-12-04T09:44:32,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T09:44:32,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38097 2024-12-04T09:44:32,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38097 2024-12-04T09:44:32,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38097 2024-12-04T09:44:32,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38097 2024-12-04T09:44:32,398 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38097 2024-12-04T09:44:32,484 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T09:44:32,485 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,485 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,485 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T09:44:32,486 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,486 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T09:44:32,488 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T09:44:32,490 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T09:44:32,491 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35147 2024-12-04T09:44:32,493 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35147 connecting to ZooKeeper ensemble=127.0.0.1:64397 2024-12-04T09:44:32,493 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,496 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351470x0, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T09:44:32,510 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35147-0x101a10432830001 connected 2024-12-04T09:44:32,510 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T09:44:32,514 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T09:44:32,523 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T09:44:32,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T09:44:32,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T09:44:32,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35147 2024-12-04T09:44:32,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35147 2024-12-04T09:44:32,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35147 2024-12-04T09:44:32,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35147 2024-12-04T09:44:32,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35147 2024-12-04T09:44:32,547 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T09:44:32,547 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,547 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,547 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T09:44:32,547 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T09:44:32,548 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T09:44:32,548 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T09:44:32,549 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38215 2024-12-04T09:44:32,550 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38215 connecting to ZooKeeper ensemble=127.0.0.1:64397 2024-12-04T09:44:32,551 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,553 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382150x0, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T09:44:32,565 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T09:44:32,565 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38215-0x101a10432830002 connected 2024-12-04T09:44:32,565 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T09:44:32,566 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T09:44:32,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T09:44:32,569 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T09:44:32,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38215 2024-12-04T09:44:32,570 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38215 2024-12-04T09:44:32,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38215 2024-12-04T09:44:32,571 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38215 2024-12-04T09:44:32,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38215 2024-12-04T09:44:32,592 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/84486a41f81c:0 server-side Connection retries=45 2024-12-04T09:44:32,593 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,593 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,593 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T09:44:32,593 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T09:44:32,593 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T09:44:32,594 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T09:44:32,594 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T09:44:32,595 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36773 2024-12-04T09:44:32,597 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36773 connecting to ZooKeeper ensemble=127.0.0.1:64397 2024-12-04T09:44:32,598 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,601 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,614 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367730x0, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T09:44:32,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36773-0x101a10432830003 connected 2024-12-04T09:44:32,615 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T09:44:32,616 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T09:44:32,616 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T09:44:32,618 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T09:44:32,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T09:44:32,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36773 2024-12-04T09:44:32,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36773 2024-12-04T09:44:32,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36773 2024-12-04T09:44:32,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36773 2024-12-04T09:44:32,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36773 2024-12-04T09:44:32,637 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;84486a41f81c:38097 2024-12-04T09:44:32,638 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/84486a41f81c,38097,1733305471718 2024-12-04T09:44:32,651 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,653 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/84486a41f81c,38097,1733305471718 2024-12-04T09:44:32,680 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T09:44:32,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T09:44:32,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T09:44:32,680 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,681 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,681 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T09:44:32,683 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/84486a41f81c,38097,1733305471718 from backup master directory 2024-12-04T09:44:32,692 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/84486a41f81c,38097,1733305471718 2024-12-04T09:44:32,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T09:44:32,694 WARN [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T09:44:32,694 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=84486a41f81c,38097,1733305471718 2024-12-04T09:44:32,697 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T09:44:32,699 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T09:44:32,751 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/hbase.id] with ID: e0ed37ab-3fa5-43a0-81c4-b1b6ab95a61c 2024-12-04T09:44:32,751 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/.tmp/hbase.id 2024-12-04T09:44:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741826_1002 (size=42) 2024-12-04T09:44:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741826_1002 (size=42) 2024-12-04T09:44:32,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741826_1002 (size=42) 2024-12-04T09:44:32,765 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/.tmp/hbase.id]:[hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/hbase.id] 2024-12-04T09:44:32,811 INFO [master/84486a41f81c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T09:44:32,816 INFO [master/84486a41f81c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T09:44:32,831 INFO [master/84486a41f81c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-04T09:44:32,842 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,842 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:32,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741827_1003 (size=196) 2024-12-04T09:44:32,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741827_1003 (size=196) 2024-12-04T09:44:32,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741827_1003 (size=196) 2024-12-04T09:44:32,877 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T09:44:32,878 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T09:44:32,883 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741828_1004 (size=1189) 2024-12-04T09:44:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741828_1004 (size=1189) 2024-12-04T09:44:32,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741828_1004 (size=1189) 2024-12-04T09:44:32,926 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store 2024-12-04T09:44:32,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741829_1005 (size=34) 2024-12-04T09:44:32,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741829_1005 (size=34) 2024-12-04T09:44:32,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741829_1005 (size=34) 2024-12-04T09:44:32,950 INFO [master/84486a41f81c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T09:44:32,954 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:32,955 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T09:44:32,955 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:44:32,956 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:44:32,957 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T09:44:32,957 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:44:32,958 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:44:32,959 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733305472955Disabling compacts and flushes for region at 1733305472955Disabling writes for close at 1733305472957 (+2 ms)Writing region close event to WAL at 1733305472958 (+1 ms)Closed at 1733305472958 2024-12-04T09:44:32,961 WARN [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/.initializing 2024-12-04T09:44:32,961 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/WALs/84486a41f81c,38097,1733305471718 2024-12-04T09:44:32,969 INFO [master/84486a41f81c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:32,982 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=84486a41f81c%2C38097%2C1733305471718, suffix=, logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/WALs/84486a41f81c,38097,1733305471718, archiveDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/oldWALs, maxLogs=10 2024-12-04T09:44:33,010 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/WALs/84486a41f81c,38097,1733305471718/84486a41f81c%2C38097%2C1733305471718.1733305472988, exclude list is [], retry=0 2024-12-04T09:44:33,026 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:33,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:33,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:33,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:33,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T09:44:33,069 INFO [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/WALs/84486a41f81c,38097,1733305471718/84486a41f81c%2C38097%2C1733305471718.1733305472988 2024-12-04T09:44:33,070 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:33,070 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:33,070 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:33,073 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,074 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T09:44:33,132 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:33,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,138 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T09:44:33,138 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:33,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T09:44:33,142 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:33,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,146 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T09:44:33,146 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:33,147 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,150 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,152 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,157 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,157 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,160 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T09:44:33,164 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T09:44:33,168 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:33,169 INFO [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62753733, jitterRate=-0.06489650905132294}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T09:44:33,174 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733305473084Initializing all the Stores at 1733305473086 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733305473086Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305473087 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305473087Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305473087Cleaning up temporary data from old regions at 1733305473157 (+70 ms)Region opened successfully at 1733305473173 (+16 ms) 2024-12-04T09:44:33,175 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T09:44:33,203 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d99bcff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T09:44:33,228 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T09:44:33,238 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T09:44:33,238 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T09:44:33,240 INFO [master/84486a41f81c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T09:44:33,241 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T09:44:33,245 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T09:44:33,246 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T09:44:33,273 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T09:44:33,282 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T09:44:33,334 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T09:44:33,340 INFO [master/84486a41f81c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T09:44:33,343 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T09:44:33,350 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T09:44:33,352 INFO [master/84486a41f81c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T09:44:33,355 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T09:44:33,363 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T09:44:33,365 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T09:44:33,372 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T09:44:33,392 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T09:44:33,400 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T09:44:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T09:44:33,414 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T09:44:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T09:44:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T09:44:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,414 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,418 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=84486a41f81c,38097,1733305471718, sessionid=0x101a10432830000, setting cluster-up flag (Was=false) 2024-12-04T09:44:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,447 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,481 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T09:44:33,486 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=84486a41f81c,38097,1733305471718 2024-12-04T09:44:33,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,509 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:33,534 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T09:44:33,536 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=84486a41f81c,38097,1733305471718 2024-12-04T09:44:33,545 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T09:44:33,613 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T09:44:33,623 INFO [master/84486a41f81c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T09:44:33,627 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(746): ClusterId : e0ed37ab-3fa5-43a0-81c4-b1b6ab95a61c 2024-12-04T09:44:33,627 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(746): ClusterId : e0ed37ab-3fa5-43a0-81c4-b1b6ab95a61c 2024-12-04T09:44:33,627 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(746): ClusterId : e0ed37ab-3fa5-43a0-81c4-b1b6ab95a61c 2024-12-04T09:44:33,629 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T09:44:33,629 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T09:44:33,629 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T09:44:33,630 INFO [master/84486a41f81c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T09:44:33,635 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 84486a41f81c,38097,1733305471718 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T09:44:33,641 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T09:44:33,641 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T09:44:33,641 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T09:44:33,641 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T09:44:33,641 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T09:44:33,641 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T09:44:33,643 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T09:44:33,643 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T09:44:33,643 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T09:44:33,643 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/84486a41f81c:0, corePoolSize=5, maxPoolSize=5 2024-12-04T09:44:33,644 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/84486a41f81c:0, corePoolSize=10, maxPoolSize=10 2024-12-04T09:44:33,644 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,644 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T09:44:33,644 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,647 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733305503647 2024-12-04T09:44:33,649 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T09:44:33,649 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T09:44:33,650 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T09:44:33,650 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T09:44:33,652 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T09:44:33,652 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T09:44:33,653 DEBUG [RS:1;84486a41f81c:38215 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f68c105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T09:44:33,653 DEBUG [RS:2;84486a41f81c:36773 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66cd2c32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T09:44:33,654 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T09:44:33,655 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T09:44:33,655 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T09:44:33,655 DEBUG [RS:0;84486a41f81c:35147 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dcaf90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=84486a41f81c/172.17.0.2:0 2024-12-04T09:44:33,655 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T09:44:33,656 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T09:44:33,657 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,657 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T09:44:33,661 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,672 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;84486a41f81c:35147 2024-12-04T09:44:33,672 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;84486a41f81c:36773 2024-12-04T09:44:33,672 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;84486a41f81c:38215 2024-12-04T09:44:33,674 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T09:44:33,675 INFO [RS:1;84486a41f81c:38215 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T09:44:33,675 INFO [RS:0;84486a41f81c:35147 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T09:44:33,675 INFO [RS:2;84486a41f81c:36773 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T09:44:33,675 INFO [RS:2;84486a41f81c:36773 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T09:44:33,675 INFO [RS:0;84486a41f81c:35147 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T09:44:33,675 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T09:44:33,675 INFO [RS:1;84486a41f81c:38215 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T09:44:33,675 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T09:44:33,675 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T09:44:33,676 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T09:44:33,676 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T09:44:33,679 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T09:44:33,679 INFO [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T09:44:33,680 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,38097,1733305471718 with port=35147, startcode=1733305472454 2024-12-04T09:44:33,680 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,38097,1733305471718 with port=38215, startcode=1733305472546 2024-12-04T09:44:33,680 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(2659): reportForDuty to master=84486a41f81c,38097,1733305471718 with port=36773, startcode=1733305472592 2024-12-04T09:44:33,687 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733305473681,5,FailOnTimeoutGroup] 2024-12-04T09:44:33,690 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733305473687,5,FailOnTimeoutGroup] 2024-12-04T09:44:33,691 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,691 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T09:44:33,691 DEBUG [RS:0;84486a41f81c:35147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T09:44:33,691 DEBUG [RS:1;84486a41f81c:38215 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T09:44:33,691 DEBUG [RS:2;84486a41f81c:36773 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T09:44:33,720 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,720 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741831_1007 (size=1321) 2024-12-04T09:44:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741831_1007 (size=1321) 2024-12-04T09:44:33,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741831_1007 (size=1321) 2024-12-04T09:44:33,724 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T09:44:33,725 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095 2024-12-04T09:44:33,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741832_1008 (size=32) 2024-12-04T09:44:33,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741832_1008 (size=32) 2024-12-04T09:44:33,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741832_1008 (size=32) 2024-12-04T09:44:33,743 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36067, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T09:44:33,743 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55815, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T09:44:33,743 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52669, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T09:44:33,743 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:33,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T09:44:33,749 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38097 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,38215,1733305472546 2024-12-04T09:44:33,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T09:44:33,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:33,751 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38097 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,38215,1733305472546 2024-12-04T09:44:33,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T09:44:33,754 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T09:44:33,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:33,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T09:44:33,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T09:44:33,759 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:33,760 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T09:44:33,763 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38097 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,35147,1733305472454 2024-12-04T09:44:33,763 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T09:44:33,763 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38097 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,35147,1733305472454 2024-12-04T09:44:33,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:33,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:33,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T09:44:33,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740 2024-12-04T09:44:33,767 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38097 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 84486a41f81c,36773,1733305472592 2024-12-04T09:44:33,767 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38097 {}] master.ServerManager(517): Registering regionserver=84486a41f81c,36773,1733305472592 2024-12-04T09:44:33,767 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740 2024-12-04T09:44:33,769 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095 2024-12-04T09:44:33,769 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33391 2024-12-04T09:44:33,769 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T09:44:33,769 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095 2024-12-04T09:44:33,769 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33391 2024-12-04T09:44:33,769 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T09:44:33,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T09:44:33,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T09:44:33,771 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095 2024-12-04T09:44:33,771 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33391 2024-12-04T09:44:33,771 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T09:44:33,772 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T09:44:33,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T09:44:33,778 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:33,779 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67597449, jitterRate=0.007280483841896057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T09:44:33,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733305473744Initializing all the Stores at 1733305473745 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733305473745Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733305473746 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305473746Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733305473746Cleaning up temporary data from old regions at 1733305473771 (+25 ms)Region opened successfully at 1733305473781 (+10 ms) 2024-12-04T09:44:33,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T09:44:33,781 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T09:44:33,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T09:44:33,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T09:44:33,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T09:44:33,782 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T09:44:33,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733305473781Disabling compacts and flushes for region at 1733305473781Disabling writes for close at 1733305473781Writing region close event to WAL at 1733305473782 (+1 ms)Closed at 1733305473782 2024-12-04T09:44:33,785 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T09:44:33,785 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T09:44:33,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T09:44:33,798 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T09:44:33,801 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T09:44:33,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T09:44:33,803 DEBUG [RS:1;84486a41f81c:38215 {}] zookeeper.ZKUtil(111): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,38215,1733305472546 2024-12-04T09:44:33,804 WARN [RS:1;84486a41f81c:38215 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T09:44:33,804 INFO [RS:1;84486a41f81c:38215 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:33,804 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546 2024-12-04T09:44:33,805 DEBUG [RS:2;84486a41f81c:36773 {}] zookeeper.ZKUtil(111): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,36773,1733305472592 2024-12-04T09:44:33,805 DEBUG [RS:0;84486a41f81c:35147 {}] zookeeper.ZKUtil(111): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/84486a41f81c,35147,1733305472454 2024-12-04T09:44:33,805 WARN [RS:2;84486a41f81c:36773 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T09:44:33,805 WARN [RS:0;84486a41f81c:35147 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T09:44:33,805 INFO [RS:0;84486a41f81c:35147 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:33,805 INFO [RS:2;84486a41f81c:36773 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:33,805 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,36773,1733305472592 2024-12-04T09:44:33,805 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,35147,1733305472454 2024-12-04T09:44:33,807 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,38215,1733305472546] 2024-12-04T09:44:33,807 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,35147,1733305472454] 2024-12-04T09:44:33,807 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [84486a41f81c,36773,1733305472592] 2024-12-04T09:44:33,829 INFO [RS:1;84486a41f81c:38215 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T09:44:33,829 INFO [RS:0;84486a41f81c:35147 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T09:44:33,829 INFO [RS:2;84486a41f81c:36773 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T09:44:33,846 INFO [RS:1;84486a41f81c:38215 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T09:44:33,846 INFO [RS:2;84486a41f81c:36773 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T09:44:33,846 INFO [RS:0;84486a41f81c:35147 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T09:44:33,851 INFO [RS:0;84486a41f81c:35147 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T09:44:33,851 INFO [RS:1;84486a41f81c:38215 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T09:44:33,851 INFO [RS:2;84486a41f81c:36773 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T09:44:33,851 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,851 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,851 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,852 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T09:44:33,854 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T09:44:33,854 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T09:44:33,857 INFO [RS:1;84486a41f81c:38215 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T09:44:33,857 INFO [RS:0;84486a41f81c:35147 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T09:44:33,857 INFO [RS:2;84486a41f81c:36773 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T09:44:33,859 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,859 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,859 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,859 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,859 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T09:44:33,860 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T09:44:33,860 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0, corePoolSize=2, maxPoolSize=2 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T09:44:33,860 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,860 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,861 DEBUG [RS:1;84486a41f81c:38215 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T09:44:33,861 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,861 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,861 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/84486a41f81c:0, corePoolSize=1, maxPoolSize=1 2024-12-04T09:44:33,861 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T09:44:33,861 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T09:44:33,861 DEBUG [RS:0;84486a41f81c:35147 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T09:44:33,861 DEBUG [RS:2;84486a41f81c:36773 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/84486a41f81c:0, corePoolSize=3, maxPoolSize=3 2024-12-04T09:44:33,863 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,864 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,864 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,864 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,864 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,864 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38215,1733305472546-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,35147,1733305472454-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,867 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,868 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,36773,1733305472592-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T09:44:33,885 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T09:44:33,887 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38215,1733305472546-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,887 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,887 INFO [RS:1;84486a41f81c:38215 {}] regionserver.Replication(171): 84486a41f81c,38215,1733305472546 started 2024-12-04T09:44:33,889 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T09:44:33,889 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T09:44:33,890 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,35147,1733305472454-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,890 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,36773,1733305472592-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,890 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,890 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,890 INFO [RS:2;84486a41f81c:36773 {}] regionserver.Replication(171): 84486a41f81c,36773,1733305472592 started 2024-12-04T09:44:33,890 INFO [RS:0;84486a41f81c:35147 {}] regionserver.Replication(171): 84486a41f81c,35147,1733305472454 started 2024-12-04T09:44:33,912 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,912 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,912 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:33,912 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,36773,1733305472592, RpcServer on 84486a41f81c/172.17.0.2:36773, sessionid=0x101a10432830003 2024-12-04T09:44:33,912 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,38215,1733305472546, RpcServer on 84486a41f81c/172.17.0.2:38215, sessionid=0x101a10432830002 2024-12-04T09:44:33,912 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1482): Serving as 84486a41f81c,35147,1733305472454, RpcServer on 84486a41f81c/172.17.0.2:35147, sessionid=0x101a10432830001 2024-12-04T09:44:33,913 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T09:44:33,913 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T09:44:33,913 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T09:44:33,913 DEBUG [RS:2;84486a41f81c:36773 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,36773,1733305472592 2024-12-04T09:44:33,913 DEBUG [RS:1;84486a41f81c:38215 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,38215,1733305472546 2024-12-04T09:44:33,913 DEBUG [RS:0;84486a41f81c:35147 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 84486a41f81c,35147,1733305472454 2024-12-04T09:44:33,914 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,35147,1733305472454' 2024-12-04T09:44:33,914 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,38215,1733305472546' 2024-12-04T09:44:33,914 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,36773,1733305472592' 2024-12-04T09:44:33,914 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T09:44:33,914 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T09:44:33,914 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T09:44:33,915 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T09:44:33,915 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T09:44:33,915 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T09:44:33,916 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T09:44:33,916 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T09:44:33,916 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T09:44:33,916 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T09:44:33,916 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T09:44:33,916 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T09:44:33,916 DEBUG [RS:2;84486a41f81c:36773 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,36773,1733305472592 2024-12-04T09:44:33,916 DEBUG [RS:0;84486a41f81c:35147 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,35147,1733305472454 2024-12-04T09:44:33,916 DEBUG [RS:1;84486a41f81c:38215 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 84486a41f81c,38215,1733305472546 2024-12-04T09:44:33,916 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,36773,1733305472592' 2024-12-04T09:44:33,916 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,35147,1733305472454' 2024-12-04T09:44:33,916 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '84486a41f81c,38215,1733305472546' 2024-12-04T09:44:33,916 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T09:44:33,916 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T09:44:33,916 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T09:44:33,917 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T09:44:33,917 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T09:44:33,917 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T09:44:33,917 DEBUG [RS:2;84486a41f81c:36773 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T09:44:33,917 DEBUG [RS:0;84486a41f81c:35147 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T09:44:33,917 INFO [RS:2;84486a41f81c:36773 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T09:44:33,917 INFO [RS:0;84486a41f81c:35147 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T09:44:33,917 DEBUG [RS:1;84486a41f81c:38215 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T09:44:33,917 INFO [RS:0;84486a41f81c:35147 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T09:44:33,917 INFO [RS:2;84486a41f81c:36773 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T09:44:33,917 INFO [RS:1;84486a41f81c:38215 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T09:44:33,917 INFO [RS:1;84486a41f81c:38215 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T09:44:33,952 WARN [84486a41f81c:38097 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T09:44:34,022 INFO [RS:0;84486a41f81c:35147 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:34,022 INFO [RS:1;84486a41f81c:38215 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:34,022 INFO [RS:2;84486a41f81c:36773 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:34,025 INFO [RS:2;84486a41f81c:36773 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=84486a41f81c%2C36773%2C1733305472592, suffix=, logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,36773,1733305472592, archiveDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs, maxLogs=32 2024-12-04T09:44:34,025 INFO [RS:0;84486a41f81c:35147 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=84486a41f81c%2C35147%2C1733305472454, suffix=, logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,35147,1733305472454, archiveDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs, maxLogs=32 2024-12-04T09:44:34,025 INFO [RS:1;84486a41f81c:38215 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=84486a41f81c%2C38215%2C1733305472546, suffix=, logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546, archiveDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs, maxLogs=32 2024-12-04T09:44:34,040 DEBUG [RS:2;84486a41f81c:36773 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,36773,1733305472592/84486a41f81c%2C36773%2C1733305472592.1733305474029, exclude list is [], retry=0 2024-12-04T09:44:34,041 DEBUG [RS:0;84486a41f81c:35147 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,35147,1733305472454/84486a41f81c%2C35147%2C1733305472454.1733305474029, exclude list is [], retry=0 2024-12-04T09:44:34,042 DEBUG [RS:1;84486a41f81c:38215 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546/84486a41f81c%2C38215%2C1733305472546.1733305474029, exclude list is [], retry=0 2024-12-04T09:44:34,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:34,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:34,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:34,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:34,046 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:34,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:34,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:34,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:34,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:34,051 INFO [RS:0;84486a41f81c:35147 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,35147,1733305472454/84486a41f81c%2C35147%2C1733305472454.1733305474029 2024-12-04T09:44:34,052 DEBUG [RS:0;84486a41f81c:35147 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:34,052 INFO [RS:1;84486a41f81c:38215 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546/84486a41f81c%2C38215%2C1733305472546.1733305474029 2024-12-04T09:44:34,052 INFO [RS:2;84486a41f81c:36773 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,36773,1733305472592/84486a41f81c%2C36773%2C1733305472592.1733305474029 2024-12-04T09:44:34,053 DEBUG [RS:1;84486a41f81c:38215 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:34,053 DEBUG [RS:2;84486a41f81c:36773 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:34,206 DEBUG [84486a41f81c:38097 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-04T09:44:34,215 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(204): Hosts are {84486a41f81c=0} racks are {/default-rack=0} 2024-12-04T09:44:34,221 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T09:44:34,222 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T09:44:34,222 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T09:44:34,222 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T09:44:34,222 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T09:44:34,222 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T09:44:34,222 INFO [84486a41f81c:38097 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T09:44:34,222 INFO [84486a41f81c:38097 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T09:44:34,222 INFO [84486a41f81c:38097 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T09:44:34,222 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T09:44:34,229 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=84486a41f81c,36773,1733305472592 2024-12-04T09:44:34,233 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 84486a41f81c,36773,1733305472592, state=OPENING 2024-12-04T09:44:34,250 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T09:44:34,259 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:34,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:34,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:34,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:44:34,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,261 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,263 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T09:44:34,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=84486a41f81c,36773,1733305472592}] 2024-12-04T09:44:34,447 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T09:44:34,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52615, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T09:44:34,460 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T09:44:34,460 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:34,461 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T09:44:34,463 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=84486a41f81c%2C36773%2C1733305472592.meta, suffix=.meta, logDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,36773,1733305472592, archiveDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs, maxLogs=32 2024-12-04T09:44:34,479 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,36773,1733305472592/84486a41f81c%2C36773%2C1733305472592.meta.1733305474465.meta, exclude list is [], retry=0 2024-12-04T09:44:34,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:34,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:34,483 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:34,486 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,36773,1733305472592/84486a41f81c%2C36773%2C1733305472592.meta.1733305474465.meta 2024-12-04T09:44:34,487 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:34,487 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:34,489 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T09:44:34,492 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T09:44:34,496 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T09:44:34,500 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T09:44:34,500 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:34,500 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T09:44:34,500 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T09:44:34,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T09:44:34,505 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T09:44:34,505 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:34,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:34,506 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T09:44:34,507 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T09:44:34,507 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:34,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:34,508 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T09:44:34,509 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T09:44:34,509 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:34,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:34,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T09:44:34,512 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T09:44:34,512 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:34,512 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T09:44:34,513 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T09:44:34,514 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740 2024-12-04T09:44:34,516 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740 2024-12-04T09:44:34,519 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T09:44:34,519 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T09:44:34,520 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T09:44:34,522 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T09:44:34,525 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61633431, jitterRate=-0.08159030973911285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T09:44:34,525 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T09:44:34,526 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733305474501Writing region info on filesystem at 1733305474501Initializing all the Stores at 1733305474503 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733305474503Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733305474503Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305474503Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733305474503Cleaning up temporary data from old regions at 1733305474519 (+16 ms)Running coprocessor post-open hooks at 1733305474525 (+6 ms)Region opened successfully at 1733305474526 (+1 ms) 2024-12-04T09:44:34,532 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733305474438 2024-12-04T09:44:34,542 DEBUG [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T09:44:34,543 INFO [RS_OPEN_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T09:44:34,544 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=84486a41f81c,36773,1733305472592 2024-12-04T09:44:34,546 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 84486a41f81c,36773,1733305472592, state=OPEN 2024-12-04T09:44:34,555 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T09:44:34,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T09:44:34,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T09:44:34,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T09:44:34,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,555 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T09:44:34,556 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=84486a41f81c,36773,1733305472592 2024-12-04T09:44:34,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T09:44:34,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=84486a41f81c,36773,1733305472592 in 291 msec 2024-12-04T09:44:34,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T09:44:34,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 774 msec 2024-12-04T09:44:34,571 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T09:44:34,571 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T09:44:34,608 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T09:44:34,609 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=84486a41f81c,36773,1733305472592, seqNum=-1] 2024-12-04T09:44:34,638 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T09:44:34,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43045, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T09:44:34,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0840 sec 2024-12-04T09:44:34,658 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733305474657, completionTime=-1 2024-12-04T09:44:34,660 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-04T09:44:34,660 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T09:44:34,681 INFO [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-04T09:44:34,682 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733305534682 2024-12-04T09:44:34,682 INFO [master/84486a41f81c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733305594682 2024-12-04T09:44:34,682 INFO [master/84486a41f81c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-04T09:44:34,683 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T09:44:34,690 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38097,1733305471718-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:34,690 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38097,1733305471718-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:34,690 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38097,1733305471718-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:34,692 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-84486a41f81c:38097, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:34,692 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:34,693 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:34,698 DEBUG [master/84486a41f81c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T09:44:34,719 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.025sec 2024-12-04T09:44:34,720 INFO [master/84486a41f81c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T09:44:34,721 INFO [master/84486a41f81c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T09:44:34,722 INFO [master/84486a41f81c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T09:44:34,722 INFO [master/84486a41f81c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T09:44:34,722 INFO [master/84486a41f81c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T09:44:34,723 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38097,1733305471718-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T09:44:34,723 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38097,1733305471718-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T09:44:34,728 DEBUG [master/84486a41f81c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T09:44:34,729 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T09:44:34,729 INFO [master/84486a41f81c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=84486a41f81c,38097,1733305471718-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:34,736 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41bb67d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T09:44:34,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 84486a41f81c,38097,-1 for getting cluster id 2024-12-04T09:44:34,740 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T09:44:34,747 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0ed37ab-3fa5-43a0-81c4-b1b6ab95a61c' 2024-12-04T09:44:34,749 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T09:44:34,749 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0ed37ab-3fa5-43a0-81c4-b1b6ab95a61c" 2024-12-04T09:44:34,749 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@905391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T09:44:34,750 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [84486a41f81c,38097,-1] 2024-12-04T09:44:34,752 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T09:44:34,754 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:44:34,755 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T09:44:34,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@402c8aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T09:44:34,759 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T09:44:34,766 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=84486a41f81c,36773,1733305472592, seqNum=-1] 2024-12-04T09:44:34,767 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T09:44:34,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T09:44:34,785 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=84486a41f81c,38097,1733305471718 2024-12-04T09:44:34,786 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33391/hbase 2024-12-04T09:44:34,797 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=361, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=11058 2024-12-04T09:44:34,815 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:34,818 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:34,819 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:34,823 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63622884, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-63622884, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:34,836 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63622884/hregion-63622884.1733305474825, exclude list is [], retry=0 2024-12-04T09:44:34,840 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:34,840 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:34,841 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:34,844 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63622884/hregion-63622884.1733305474825 2024-12-04T09:44:34,844 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:34,845 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f6d7126823ce6d01fcc4ae899649bbff, NAME => 'testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:34,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741838_1014 (size=64) 2024-12-04T09:44:34,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741838_1014 (size=64) 2024-12-04T09:44:34,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741838_1014 (size=64) 2024-12-04T09:44:34,860 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:34,863 INFO [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:34,865 INFO [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6d7126823ce6d01fcc4ae899649bbff columnFamilyName a 2024-12-04T09:44:34,865 DEBUG [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:34,866 INFO [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] regionserver.HStore(327): Store=f6d7126823ce6d01fcc4ae899649bbff/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:34,867 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:34,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:34,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:34,869 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:34,869 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:34,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:34,875 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:34,876 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f6d7126823ce6d01fcc4ae899649bbff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63895466, jitterRate=-0.04788336157798767}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:44:34,877 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f6d7126823ce6d01fcc4ae899649bbff: Writing region info on filesystem at 1733305474861Initializing all the Stores at 1733305474862 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305474862Cleaning up temporary data from old regions at 1733305474869 (+7 ms)Region opened successfully at 1733305474877 (+8 ms) 2024-12-04T09:44:34,877 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f6d7126823ce6d01fcc4ae899649bbff, disabling compactions & flushes 2024-12-04T09:44:34,877 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff. 2024-12-04T09:44:34,877 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff. 2024-12-04T09:44:34,877 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff. after waiting 0 ms 2024-12-04T09:44:34,877 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff. 2024-12-04T09:44:34,878 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff. 2024-12-04T09:44:34,878 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f6d7126823ce6d01fcc4ae899649bbff: Waiting for close lock at 1733305474877Disabling compacts and flushes for region at 1733305474877Disabling writes for close at 1733305474877Writing region close event to WAL at 1733305474878 (+1 ms)Closed at 1733305474878 2024-12-04T09:44:34,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741837_1013 (size=93) 2024-12-04T09:44:34,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741837_1013 (size=93) 2024-12-04T09:44:34,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741837_1013 (size=93) 2024-12-04T09:44:34,889 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:34,890 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63622884:(num 1733305474825) 2024-12-04T09:44:34,892 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T09:44:34,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741839_1015 (size=276) 2024-12-04T09:44:34,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741839_1015 (size=276) 2024-12-04T09:44:34,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741839_1015 (size=276) 2024-12-04T09:44:34,911 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T09:44:34,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741840_1016 (size=230) 2024-12-04T09:44:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741840_1016 (size=230) 2024-12-04T09:44:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741840_1016 (size=230) 2024-12-04T09:44:34,943 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1, size=276 (276bytes) 2024-12-04T09:44:34,943 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T09:44:34,944 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T09:44:34,944 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1 2024-12-04T09:44:34,949 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1 after 3ms 2024-12-04T09:44:34,954 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:34,955 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1 took 14ms 2024-12-04T09:44:34,960 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1 so closing down 2024-12-04T09:44:34,962 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T09:44:34,964 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T09:44:34,965 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:34,966 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:34,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741841_1017 (size=276) 2024-12-04T09:44:34,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741841_1017 (size=276) 2024-12-04T09:44:34,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741841_1017 (size=276) 2024-12-04T09:44:34,978 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:34,981 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002 2024-12-04T09:44:34,986 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-04T09:44:34,986 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1, journal: Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1, size=276 (276bytes) at 1733305474943Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1 so closing down at 1733305474960 (+17 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000001-wal-1.temp at 1733305474964 (+4 ms)3 split writer threads finished at 1733305474966 (+2 ms)Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733305474978 (+12 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002 at 1733305474981 (+3 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733305474986 (+5 ms) 2024-12-04T09:44:35,000 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2, size=230 (230bytes) 2024-12-04T09:44:35,000 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2 2024-12-04T09:44:35,001 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2 after 1ms 2024-12-04T09:44:35,005 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:35,005 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2 took 5ms 2024-12-04T09:44:35,008 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2 so closing down 2024-12-04T09:44:35,008 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:35,011 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T09:44:35,013 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T09:44:35,013 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:35,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741842_1018 (size=230) 2024-12-04T09:44:35,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741842_1018 (size=230) 2024-12-04T09:44:35,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741842_1018 (size=230) 2024-12-04T09:44:35,022 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:35,026 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:35,028 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-04T09:44:35,029 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-04T09:44:35,029 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2, journal: Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2, size=230 (230bytes) at 1733305475000Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2 so closing down at 1733305475008 (+8 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002-wal-2.temp at 1733305475013 (+5 ms)3 split writer threads finished at 1733305475013Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733305475022 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733305475029 (+7 ms) 2024-12-04T09:44:35,030 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:35,032 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:35,047 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal.1733305475034, exclude list is [], retry=0 2024-12-04T09:44:35,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:35,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:35,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:35,054 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal.1733305475034 2024-12-04T09:44:35,054 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:35,054 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f6d7126823ce6d01fcc4ae899649bbff, NAME => 'testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:35,055 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:35,055 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,055 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,057 INFO [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,058 INFO [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6d7126823ce6d01fcc4ae899649bbff columnFamilyName a 2024-12-04T09:44:35,058 DEBUG [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:35,059 INFO [StoreOpener-f6d7126823ce6d01fcc4ae899649bbff-1 {}] regionserver.HStore(327): Store=f6d7126823ce6d01fcc4ae899649bbff/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:35,060 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,061 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,066 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002 2024-12-04T09:44:35,070 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:35,076 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002 2024-12-04T09:44:35,079 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f6d7126823ce6d01fcc4ae899649bbff 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T09:44:35,135 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/.tmp/a/a7fd4f6733f54b2fb97f9225160dc008 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733305474890/Put/seqid=0 2024-12-04T09:44:35,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741844_1020 (size=5170) 2024-12-04T09:44:35,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741844_1020 (size=5170) 2024-12-04T09:44:35,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741844_1020 (size=5170) 2024-12-04T09:44:35,154 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/.tmp/a/a7fd4f6733f54b2fb97f9225160dc008 2024-12-04T09:44:35,195 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/.tmp/a/a7fd4f6733f54b2fb97f9225160dc008 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/a/a7fd4f6733f54b2fb97f9225160dc008 2024-12-04T09:44:35,205 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/a/a7fd4f6733f54b2fb97f9225160dc008, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T09:44:35,210 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f6d7126823ce6d01fcc4ae899649bbff in 130ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T09:44:35,212 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/0000000000000000002 2024-12-04T09:44:35,212 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,213 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,216 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f6d7126823ce6d01fcc4ae899649bbff 2024-12-04T09:44:35,219 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/f6d7126823ce6d01fcc4ae899649bbff/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T09:44:35,220 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f6d7126823ce6d01fcc4ae899649bbff; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74261988, jitterRate=0.10658985376358032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:44:35,221 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f6d7126823ce6d01fcc4ae899649bbff: Writing region info on filesystem at 1733305475055Initializing all the Stores at 1733305475056 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305475056Obtaining lock to block concurrent updates at 1733305475079 (+23 ms)Preparing flush snapshotting stores in f6d7126823ce6d01fcc4ae899649bbff at 1733305475079Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733305475082 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733305474816.f6d7126823ce6d01fcc4ae899649bbff. at 1733305475082Flushing f6d7126823ce6d01fcc4ae899649bbff/a: creating writer at 1733305475083 (+1 ms)Flushing f6d7126823ce6d01fcc4ae899649bbff/a: appending metadata at 1733305475124 (+41 ms)Flushing f6d7126823ce6d01fcc4ae899649bbff/a: closing flushed file at 1733305475127 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f449ecc: reopening flushed file at 1733305475193 (+66 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f6d7126823ce6d01fcc4ae899649bbff in 130ms, sequenceid=2, compaction requested=false; wal=null at 1733305475210 (+17 ms)Cleaning up temporary data from old regions at 1733305475213 (+3 ms)Region opened successfully at 1733305475221 (+8 ms) 2024-12-04T09:44:35,245 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=372 (was 361) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:50708 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:57630 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:50850 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:59276 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:59452 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33391/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=460 (was 460), ProcessCount=11 (was 11), AvailableMemoryMB=11045 (was 11058) 2024-12-04T09:44:35,254 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=372, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=11045 2024-12-04T09:44:35,268 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:35,270 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:35,271 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:35,274 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-24874718, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-24874718, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:35,285 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-24874718/hregion-24874718.1733305475275, exclude list is [], retry=0 2024-12-04T09:44:35,289 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:35,289 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:35,289 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:35,292 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-24874718/hregion-24874718.1733305475275 2024-12-04T09:44:35,292 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:35,292 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 12d680d8b168ecc7ee311b7ea2c13e6c, NAME => 'testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741846_1022 (size=64) 2024-12-04T09:44:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741846_1022 (size=64) 2024-12-04T09:44:35,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741846_1022 (size=64) 2024-12-04T09:44:35,306 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:35,308 INFO [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,310 INFO [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12d680d8b168ecc7ee311b7ea2c13e6c columnFamilyName a 2024-12-04T09:44:35,310 DEBUG [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:35,311 INFO [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] regionserver.HStore(327): Store=12d680d8b168ecc7ee311b7ea2c13e6c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:35,311 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,313 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,314 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,314 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,317 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,320 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:35,321 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12d680d8b168ecc7ee311b7ea2c13e6c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59874768, jitterRate=-0.10779643058776855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:44:35,322 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12d680d8b168ecc7ee311b7ea2c13e6c: Writing region info on filesystem at 1733305475306Initializing all the Stores at 1733305475307 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305475308 (+1 ms)Cleaning up temporary data from old regions at 1733305475314 (+6 ms)Region opened successfully at 1733305475321 (+7 ms) 2024-12-04T09:44:35,322 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 12d680d8b168ecc7ee311b7ea2c13e6c, disabling compactions & flushes 2024-12-04T09:44:35,322 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c. 2024-12-04T09:44:35,322 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c. 2024-12-04T09:44:35,322 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c. after waiting 0 ms 2024-12-04T09:44:35,322 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c. 2024-12-04T09:44:35,323 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c. 2024-12-04T09:44:35,323 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 12d680d8b168ecc7ee311b7ea2c13e6c: Waiting for close lock at 1733305475322Disabling compacts and flushes for region at 1733305475322Disabling writes for close at 1733305475322Writing region close event to WAL at 1733305475322Closed at 1733305475322 2024-12-04T09:44:35,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741845_1021 (size=93) 2024-12-04T09:44:35,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741845_1021 (size=93) 2024-12-04T09:44:35,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741845_1021 (size=93) 2024-12-04T09:44:35,329 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:35,329 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-24874718:(num 1733305475275) 2024-12-04T09:44:35,330 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T09:44:35,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741847_1023 (size=276) 2024-12-04T09:44:35,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741847_1023 (size=276) 2024-12-04T09:44:35,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741847_1023 (size=276) 2024-12-04T09:44:35,343 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T09:44:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741848_1024 (size=230) 2024-12-04T09:44:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741848_1024 (size=230) 2024-12-04T09:44:35,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741848_1024 (size=230) 2024-12-04T09:44:35,367 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2, size=230 (230bytes) 2024-12-04T09:44:35,367 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2 2024-12-04T09:44:35,368 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2 after 1ms 2024-12-04T09:44:35,372 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:35,373 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2 took 5ms 2024-12-04T09:44:35,375 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2 so closing down 2024-12-04T09:44:35,375 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:35,377 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T09:44:35,379 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T09:44:35,379 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741849_1025 (size=230) 2024-12-04T09:44:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741849_1025 (size=230) 2024-12-04T09:44:35,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741849_1025 (size=230) 2024-12-04T09:44:35,387 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:35,389 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002 2024-12-04T09:44:35,389 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-04T09:44:35,390 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2, journal: Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2, size=230 (230bytes) at 1733305475367Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2 so closing down at 1733305475375 (+8 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002-wal-2.temp at 1733305475379 (+4 ms)3 split writer threads finished at 1733305475379Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733305475387 (+8 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002 at 1733305475389 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733305475389 2024-12-04T09:44:35,402 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1, size=276 (276bytes) 2024-12-04T09:44:35,403 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1 2024-12-04T09:44:35,403 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1 after 0ms 2024-12-04T09:44:35,407 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:35,407 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1 took 5ms 2024-12-04T09:44:35,410 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1 so closing down 2024-12-04T09:44:35,410 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:35,412 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T09:44:35,413 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T09:44:35,414 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:35,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741850_1026 (size=276) 2024-12-04T09:44:35,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741850_1026 (size=276) 2024-12-04T09:44:35,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741850_1026 (size=276) 2024-12-04T09:44:35,422 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:35,426 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:35,428 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002, length=230 2024-12-04T09:44:35,431 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002 2024-12-04T09:44:35,431 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-04T09:44:35,431 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1, journal: Splitting hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1, size=276 (276bytes) at 1733305475403Finishing writing output for hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1 so closing down at 1733305475410 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000001-wal-1.temp at 1733305475413 (+3 ms)3 split writer threads finished at 1733305475414 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733305475422 (+8 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002 at 1733305475431 (+9 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733305475431 2024-12-04T09:44:35,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:35,433 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:35,445 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal.1733305475434, exclude list is [], retry=0 2024-12-04T09:44:35,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:35,449 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:35,450 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:35,452 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal.1733305475434 2024-12-04T09:44:35,452 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:35,452 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 12d680d8b168ecc7ee311b7ea2c13e6c, NAME => 'testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:35,453 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:35,453 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,453 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,455 INFO [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,457 INFO [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12d680d8b168ecc7ee311b7ea2c13e6c columnFamilyName a 2024-12-04T09:44:35,457 DEBUG [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:35,458 INFO [StoreOpener-12d680d8b168ecc7ee311b7ea2c13e6c-1 {}] regionserver.HStore(327): Store=12d680d8b168ecc7ee311b7ea2c13e6c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:35,458 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,459 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,461 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,462 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002 2024-12-04T09:44:35,465 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:35,467 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002 2024-12-04T09:44:35,467 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 12d680d8b168ecc7ee311b7ea2c13e6c 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T09:44:35,482 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/.tmp/a/8a3516eb7ba74b85a9cce6da922c075e is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733305475329/Put/seqid=0 2024-12-04T09:44:35,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741852_1028 (size=5170) 2024-12-04T09:44:35,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741852_1028 (size=5170) 2024-12-04T09:44:35,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741852_1028 (size=5170) 2024-12-04T09:44:35,492 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/.tmp/a/8a3516eb7ba74b85a9cce6da922c075e 2024-12-04T09:44:35,501 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/.tmp/a/8a3516eb7ba74b85a9cce6da922c075e as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/a/8a3516eb7ba74b85a9cce6da922c075e 2024-12-04T09:44:35,509 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/a/8a3516eb7ba74b85a9cce6da922c075e, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T09:44:35,509 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 12d680d8b168ecc7ee311b7ea2c13e6c in 42ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T09:44:35,510 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/0000000000000000002 2024-12-04T09:44:35,511 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,511 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,513 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12d680d8b168ecc7ee311b7ea2c13e6c 2024-12-04T09:44:35,516 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/12d680d8b168ecc7ee311b7ea2c13e6c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T09:44:35,517 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12d680d8b168ecc7ee311b7ea2c13e6c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62550291, jitterRate=-0.06792803108692169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:44:35,518 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12d680d8b168ecc7ee311b7ea2c13e6c: Writing region info on filesystem at 1733305475453Initializing all the Stores at 1733305475454 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305475454Obtaining lock to block concurrent updates at 1733305475467 (+13 ms)Preparing flush snapshotting stores in 12d680d8b168ecc7ee311b7ea2c13e6c at 1733305475467Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733305475467Flushing stores of testReplayEditsWrittenIntoWAL,,1733305475269.12d680d8b168ecc7ee311b7ea2c13e6c. at 1733305475467Flushing 12d680d8b168ecc7ee311b7ea2c13e6c/a: creating writer at 1733305475467Flushing 12d680d8b168ecc7ee311b7ea2c13e6c/a: appending metadata at 1733305475481 (+14 ms)Flushing 12d680d8b168ecc7ee311b7ea2c13e6c/a: closing flushed file at 1733305475481Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@403a0f07: reopening flushed file at 1733305475499 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 12d680d8b168ecc7ee311b7ea2c13e6c in 42ms, sequenceid=2, compaction requested=false; wal=null at 1733305475509 (+10 ms)Cleaning up temporary data from old regions at 1733305475511 (+2 ms)Region opened successfully at 1733305475518 (+7 ms) 2024-12-04T09:44:35,537 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=382 (was 372) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:50708 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:59520 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:57530 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:59276 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:50922 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:57704 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=460 (was 460), ProcessCount=11 (was 11), AvailableMemoryMB=11040 (was 11045) 2024-12-04T09:44:35,548 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=382, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=460, ProcessCount=11, AvailableMemoryMB=11039 2024-12-04T09:44:35,561 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:35,563 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:35,563 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:35,566 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85966985, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-85966985, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:35,579 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85966985/hregion-85966985.1733305475567, exclude list is [], retry=0 2024-12-04T09:44:35,582 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:35,583 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:35,583 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:35,585 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85966985/hregion-85966985.1733305475567 2024-12-04T09:44:35,585 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:35,585 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a71be46735492cef863fff108a9df3c8, NAME => 'testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:35,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741854_1030 (size=64) 2024-12-04T09:44:35,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741854_1030 (size=64) 2024-12-04T09:44:35,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741854_1030 (size=64) 2024-12-04T09:44:35,596 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:35,598 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,600 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71be46735492cef863fff108a9df3c8 columnFamilyName a 2024-12-04T09:44:35,600 DEBUG [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:35,600 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(327): Store=a71be46735492cef863fff108a9df3c8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:35,601 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,602 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71be46735492cef863fff108a9df3c8 columnFamilyName b 2024-12-04T09:44:35,602 DEBUG [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:35,603 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(327): Store=a71be46735492cef863fff108a9df3c8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:35,603 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,604 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71be46735492cef863fff108a9df3c8 columnFamilyName c 2024-12-04T09:44:35,604 DEBUG [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:35,605 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(327): Store=a71be46735492cef863fff108a9df3c8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:35,605 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,606 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,608 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,608 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,609 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:35,610 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:35,612 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:35,613 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a71be46735492cef863fff108a9df3c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64604969, jitterRate=-0.03731094300746918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:35,613 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a71be46735492cef863fff108a9df3c8: Writing region info on filesystem at 1733305475596Initializing all the Stores at 1733305475597 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305475597Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305475598 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305475598Cleaning up temporary data from old regions at 1733305475608 (+10 ms)Region opened successfully at 1733305475613 (+5 ms) 2024-12-04T09:44:35,613 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a71be46735492cef863fff108a9df3c8, disabling compactions & flushes 2024-12-04T09:44:35,613 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:35,613 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:35,613 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. after waiting 0 ms 2024-12-04T09:44:35,613 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:35,614 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:35,614 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a71be46735492cef863fff108a9df3c8: Waiting for close lock at 1733305475613Disabling compacts and flushes for region at 1733305475613Disabling writes for close at 1733305475613Writing region close event to WAL at 1733305475614 (+1 ms)Closed at 1733305475614 2024-12-04T09:44:35,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741853_1029 (size=93) 2024-12-04T09:44:35,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741853_1029 (size=93) 2024-12-04T09:44:35,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741853_1029 (size=93) 2024-12-04T09:44:35,620 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:35,620 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85966985:(num 1733305475567) 2024-12-04T09:44:35,621 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:35,623 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:35,639 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624, exclude list is [], retry=0 2024-12-04T09:44:35,643 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:35,643 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:35,644 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:35,646 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 2024-12-04T09:44:35,646 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:35,815 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624, size=0 (0bytes) 2024-12-04T09:44:35,815 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 might be still open, length is 0 2024-12-04T09:44:35,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 2024-12-04T09:44:35,816 WARN [IPC Server handler 1 on default port 33391 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-04T09:44:35,817 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 after 2ms 2024-12-04T09:44:36,834 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:57734 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36275:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57734 dst: /127.0.0.1:36275 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36275 remote=/127.0.0.1:57734]. Total timeout mills is 60000, 58948 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:36,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:50954 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33979:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50954 dst: /127.0.0.1:33979 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:36,836 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:59560 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59560 dst: /127.0.0.1:40851 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:36,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741855_1032 (size=200589) 2024-12-04T09:44:36,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741855_1032 (size=200589) 2024-12-04T09:44:36,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741855_1032 (size=200589) 2024-12-04T09:44:39,818 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 after 4003ms 2024-12-04T09:44:39,822 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:39,824 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 took 4010ms 2024-12-04T09:44:39,829 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733305475624.temp 2024-12-04T09:44:39,834 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000000001-wal.1733305475624.temp 2024-12-04T09:44:39,939 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624; continuing. 2024-12-04T09:44:39,939 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 so closing down 2024-12-04T09:44:39,939 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:39,939 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:39,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741856_1033 (size=200597) 2024-12-04T09:44:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741856_1033 (size=200597) 2024-12-04T09:44:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741856_1033 (size=200597) 2024-12-04T09:44:39,944 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000000001-wal.1733305475624.temp (wrote 3002 edits, skipped 0 edits in 37 ms) 2024-12-04T09:44:39,946 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000000001-wal.1733305475624.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000003002 2024-12-04T09:44:39,946 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 122 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T09:44:39,946 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624, journal: Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624, size=0 (0bytes) at 1733305475815Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000000001-wal.1733305475624.temp at 1733305479834 (+4019 ms)Split 1024 edits, skipped 0 edits. at 1733305479888 (+54 ms)Split 2048 edits, skipped 0 edits. at 1733305479914 (+26 ms)Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 so closing down at 1733305479939 (+25 ms)3 split writer threads finished at 1733305479939Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000000001-wal.1733305475624.temp (wrote 3002 edits, skipped 0 edits in 37 ms) at 1733305479944 (+5 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000000001-wal.1733305475624.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000003002 at 1733305479946 (+2 ms)Processed 3002 edits across 1 Regions in 122 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624, size=0, length=0, corrupted=false, cancelled=false at 1733305479946 2024-12-04T09:44:39,949 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305475624 2024-12-04T09:44:39,950 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000003002 2024-12-04T09:44:39,951 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:39,953 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:39,968 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305479954, exclude list is [], retry=0 2024-12-04T09:44:39,971 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:39,972 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:39,972 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:39,974 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305479954 2024-12-04T09:44:39,974 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:44:39,975 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:39,976 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:39,977 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71be46735492cef863fff108a9df3c8 columnFamilyName a 2024-12-04T09:44:39,977 DEBUG [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:39,978 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(327): Store=a71be46735492cef863fff108a9df3c8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:39,978 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:39,979 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71be46735492cef863fff108a9df3c8 columnFamilyName b 2024-12-04T09:44:39,979 DEBUG [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:39,979 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(327): Store=a71be46735492cef863fff108a9df3c8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:39,979 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:39,980 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a71be46735492cef863fff108a9df3c8 columnFamilyName c 2024-12-04T09:44:39,980 DEBUG [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:39,981 INFO [StoreOpener-a71be46735492cef863fff108a9df3c8-1 {}] regionserver.HStore(327): Store=a71be46735492cef863fff108a9df3c8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:39,981 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:39,982 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:39,983 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:39,984 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000003002 2024-12-04T09:44:39,987 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:40,013 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T09:44:40,019 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T09:44:40,076 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T09:44:40,375 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a71be46735492cef863fff108a9df3c8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T09:44:40,405 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/a/4eeb13f6f4d5427b970508d95b6b378c is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733305475654/Put/seqid=0 2024-12-04T09:44:40,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741858_1035 (size=50463) 2024-12-04T09:44:40,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741858_1035 (size=50463) 2024-12-04T09:44:40,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741858_1035 (size=50463) 2024-12-04T09:44:40,414 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/a/4eeb13f6f4d5427b970508d95b6b378c 2024-12-04T09:44:40,422 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/a/4eeb13f6f4d5427b970508d95b6b378c as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/a/4eeb13f6f4d5427b970508d95b6b378c 2024-12-04T09:44:40,430 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/a/4eeb13f6f4d5427b970508d95b6b378c, entries=754, sequenceid=754, filesize=49.3 K 2024-12-04T09:44:40,431 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for a71be46735492cef863fff108a9df3c8 in 56ms, sequenceid=754, compaction requested=false; wal=null 2024-12-04T09:44:40,452 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T09:44:40,452 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a71be46735492cef863fff108a9df3c8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T09:44:40,462 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/a/2a667a85da7d48719807f596e7c7cd98 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733305475688/Put/seqid=0 2024-12-04T09:44:40,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741859_1036 (size=20072) 2024-12-04T09:44:40,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741859_1036 (size=20072) 2024-12-04T09:44:40,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741859_1036 (size=20072) 2024-12-04T09:44:40,474 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/a/2a667a85da7d48719807f596e7c7cd98 2024-12-04T09:44:40,502 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/b/247aa786eb6d47dda4372b0ab3b29401 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733305475706/Put/seqid=0 2024-12-04T09:44:40,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741860_1037 (size=35835) 2024-12-04T09:44:40,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741860_1037 (size=35835) 2024-12-04T09:44:40,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741860_1037 (size=35835) 2024-12-04T09:44:40,511 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/b/247aa786eb6d47dda4372b0ab3b29401 2024-12-04T09:44:40,520 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/a/2a667a85da7d48719807f596e7c7cd98 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/a/2a667a85da7d48719807f596e7c7cd98 2024-12-04T09:44:40,527 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/a/2a667a85da7d48719807f596e7c7cd98, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-04T09:44:40,529 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/b/247aa786eb6d47dda4372b0ab3b29401 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/b/247aa786eb6d47dda4372b0ab3b29401 2024-12-04T09:44:40,536 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/b/247aa786eb6d47dda4372b0ab3b29401, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-04T09:44:40,536 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a71be46735492cef863fff108a9df3c8 in 84ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-04T09:44:40,546 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T09:44:40,547 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a71be46735492cef863fff108a9df3c8 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T09:44:40,553 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/b/2f9131b7444b412f8d340d7c446b9f0b is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733305475721/Put/seqid=0 2024-12-04T09:44:40,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741861_1038 (size=35082) 2024-12-04T09:44:40,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741861_1038 (size=35082) 2024-12-04T09:44:40,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741861_1038 (size=35082) 2024-12-04T09:44:40,562 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/b/2f9131b7444b412f8d340d7c446b9f0b 2024-12-04T09:44:40,588 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/c/3562a5223c3f407ab95a81242fe23764 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733305475746/Put/seqid=0 2024-12-04T09:44:40,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741862_1039 (size=20825) 2024-12-04T09:44:40,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741862_1039 (size=20825) 2024-12-04T09:44:40,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741862_1039 (size=20825) 2024-12-04T09:44:40,597 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/c/3562a5223c3f407ab95a81242fe23764 2024-12-04T09:44:40,606 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/b/2f9131b7444b412f8d340d7c446b9f0b as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/b/2f9131b7444b412f8d340d7c446b9f0b 2024-12-04T09:44:40,616 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/b/2f9131b7444b412f8d340d7c446b9f0b, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-04T09:44:40,617 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/c/3562a5223c3f407ab95a81242fe23764 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/c/3562a5223c3f407ab95a81242fe23764 2024-12-04T09:44:40,626 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/c/3562a5223c3f407ab95a81242fe23764, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-04T09:44:40,626 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a71be46735492cef863fff108a9df3c8 in 79ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-04T09:44:40,643 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733305475782/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:40,647 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000003002 2024-12-04T09:44:40,647 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T09:44:40,648 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a71be46735492cef863fff108a9df3c8 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-04T09:44:40,658 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/c/3ad5236137c846f8bd2cf5a89bb0b451 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733305475751/Put/seqid=0 2024-12-04T09:44:40,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741863_1040 (size=50301) 2024-12-04T09:44:40,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741863_1040 (size=50301) 2024-12-04T09:44:40,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741863_1040 (size=50301) 2024-12-04T09:44:40,668 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/c/3ad5236137c846f8bd2cf5a89bb0b451 2024-12-04T09:44:40,675 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3ad5236137c846f8bd2cf5a89bb0b451 2024-12-04T09:44:40,676 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/.tmp/c/3ad5236137c846f8bd2cf5a89bb0b451 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/c/3ad5236137c846f8bd2cf5a89bb0b451 2024-12-04T09:44:40,683 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3ad5236137c846f8bd2cf5a89bb0b451 2024-12-04T09:44:40,683 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/c/3ad5236137c846f8bd2cf5a89bb0b451, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-04T09:44:40,683 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for a71be46735492cef863fff108a9df3c8 in 36ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-04T09:44:40,684 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/0000000000000003002 2024-12-04T09:44:40,686 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:40,686 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:40,687 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T09:44:40,688 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a71be46735492cef863fff108a9df3c8 2024-12-04T09:44:40,691 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenIntoWAL/a71be46735492cef863fff108a9df3c8/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-04T09:44:40,692 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a71be46735492cef863fff108a9df3c8; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64815488, jitterRate=-0.03417396545410156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T09:44:40,692 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a71be46735492cef863fff108a9df3c8: Writing region info on filesystem at 1733305479975Initializing all the Stores at 1733305479976 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305479976Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305479976Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305479976Cleaning up temporary data from old regions at 1733305480686 (+710 ms)Region opened successfully at 1733305480692 (+6 ms) 2024-12-04T09:44:40,752 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a71be46735492cef863fff108a9df3c8, disabling compactions & flushes 2024-12-04T09:44:40,752 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:40,752 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:40,752 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. after waiting 0 ms 2024-12-04T09:44:40,752 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:40,754 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733305475561.a71be46735492cef863fff108a9df3c8. 2024-12-04T09:44:40,754 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a71be46735492cef863fff108a9df3c8: Waiting for close lock at 1733305480751Disabling compacts and flushes for region at 1733305480751Disabling writes for close at 1733305480752 (+1 ms)Writing region close event to WAL at 1733305480754 (+2 ms)Closed at 1733305480754 2024-12-04T09:44:40,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741857_1034 (size=93) 2024-12-04T09:44:40,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741857_1034 (size=93) 2024-12-04T09:44:40,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741857_1034 (size=93) 2024-12-04T09:44:40,761 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:40,761 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733305479954) 2024-12-04T09:44:40,778 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=398 (was 382) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36281 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:45719 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1894355275_22 at /127.0.0.1:44154 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33391 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:33391 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45719 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1894355275_22 at /127.0.0.1:60250 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1894355275_22 at /127.0.0.1:60230 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1894355275_22 at /127.0.0.1:34756 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:36281 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 460), ProcessCount=11 (was 11), AvailableMemoryMB=10947 (was 11039) 2024-12-04T09:44:40,788 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=398, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=423, ProcessCount=11, AvailableMemoryMB=10946 2024-12-04T09:44:40,803 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:40,805 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:40,806 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:40,809 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-25261761, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-25261761, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:40,821 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-25261761/hregion-25261761.1733305480809, exclude list is [], retry=0 2024-12-04T09:44:40,824 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:40,824 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:40,824 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:40,826 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-25261761/hregion-25261761.1733305480809 2024-12-04T09:44:40,826 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:40,826 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0854c8f4d5538dd9a2ef75512d623e64, NAME => 'test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741865_1042 (size=43) 2024-12-04T09:44:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741865_1042 (size=43) 2024-12-04T09:44:40,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741865_1042 (size=43) 2024-12-04T09:44:40,836 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:40,837 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,839 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0854c8f4d5538dd9a2ef75512d623e64 columnFamilyName a 2024-12-04T09:44:40,839 DEBUG [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:40,840 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(327): Store=0854c8f4d5538dd9a2ef75512d623e64/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:40,840 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,842 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0854c8f4d5538dd9a2ef75512d623e64 columnFamilyName b 2024-12-04T09:44:40,842 DEBUG [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:40,842 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(327): Store=0854c8f4d5538dd9a2ef75512d623e64/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:40,843 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,844 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0854c8f4d5538dd9a2ef75512d623e64 columnFamilyName c 2024-12-04T09:44:40,845 DEBUG [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:40,845 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(327): Store=0854c8f4d5538dd9a2ef75512d623e64/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:40,845 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,846 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,847 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,848 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,848 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,849 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:40,850 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:40,853 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:40,854 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0854c8f4d5538dd9a2ef75512d623e64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74205016, jitterRate=0.10574090480804443}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:40,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0854c8f4d5538dd9a2ef75512d623e64: Writing region info on filesystem at 1733305480836Initializing all the Stores at 1733305480837 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305480837Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305480837Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305480837Cleaning up temporary data from old regions at 1733305480848 (+11 ms)Region opened successfully at 1733305480855 (+7 ms) 2024-12-04T09:44:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0854c8f4d5538dd9a2ef75512d623e64, disabling compactions & flushes 2024-12-04T09:44:40,856 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. after waiting 0 ms 2024-12-04T09:44:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:40,856 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:40,856 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0854c8f4d5538dd9a2ef75512d623e64: Waiting for close lock at 1733305480855Disabling compacts and flushes for region at 1733305480855Disabling writes for close at 1733305480856 (+1 ms)Writing region close event to WAL at 1733305480856Closed at 1733305480856 2024-12-04T09:44:40,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741864_1041 (size=93) 2024-12-04T09:44:40,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741864_1041 (size=93) 2024-12-04T09:44:40,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741864_1041 (size=93) 2024-12-04T09:44:40,862 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:40,862 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-25261761:(num 1733305480809) 2024-12-04T09:44:40,862 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:40,865 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:40,881 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865, exclude list is [], retry=0 2024-12-04T09:44:40,884 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:40,884 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:40,885 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:40,887 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865 2024-12-04T09:44:40,888 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:41,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741866_1043 (size=200357) 2024-12-04T09:44:41,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741866_1043 (size=200357) 2024-12-04T09:44:41,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741866_1043 (size=200357) 2024-12-04T09:44:41,071 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865, size=195.7 K (200357bytes) 2024-12-04T09:44:41,071 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865 2024-12-04T09:44:41,072 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865 after 1ms 2024-12-04T09:44:41,075 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:41,077 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865 took 6ms 2024-12-04T09:44:41,081 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733305480865.temp 2024-12-04T09:44:41,084 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000000001-wal.1733305480865.temp 2024-12-04T09:44:41,125 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865 so closing down 2024-12-04T09:44:41,125 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:41,125 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:41,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741867_1044 (size=200357) 2024-12-04T09:44:41,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741867_1044 (size=200357) 2024-12-04T09:44:41,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741867_1044 (size=200357) 2024-12-04T09:44:41,129 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000000001-wal.1733305480865.temp (wrote 3000 edits, skipped 0 edits in 18 ms) 2024-12-04T09:44:41,131 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000000001-wal.1733305480865.temp to hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003000 2024-12-04T09:44:41,131 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-04T09:44:41,131 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865, journal: Splitting hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865, size=195.7 K (200357bytes) at 1733305481071Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000000001-wal.1733305480865.temp at 1733305481084 (+13 ms)Split 1024 edits, skipped 0 edits. at 1733305481094 (+10 ms)Split 2048 edits, skipped 0 edits. at 1733305481112 (+18 ms)Finishing writing output for hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865 so closing down at 1733305481125 (+13 ms)3 split writer threads finished at 1733305481125Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000000001-wal.1733305480865.temp (wrote 3000 edits, skipped 0 edits in 18 ms) at 1733305481129 (+4 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000000001-wal.1733305480865.temp to hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003000 at 1733305481131 (+2 ms)Processed 3000 edits across 1 Regions in 53 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733305481131 2024-12-04T09:44:41,134 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305480865 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305480865 2024-12-04T09:44:41,135 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003000 2024-12-04T09:44:41,135 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:41,137 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:41,150 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138, exclude list is [], retry=0 2024-12-04T09:44:41,153 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:41,154 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:41,154 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:41,157 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138 2024-12-04T09:44:41,157 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:44:41,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741868_1045 (size=200484) 2024-12-04T09:44:41,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741868_1045 (size=200484) 2024-12-04T09:44:41,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741868_1045 (size=200484) 2024-12-04T09:44:41,316 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138, size=195.8 K (200484bytes) 2024-12-04T09:44:41,316 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138 2024-12-04T09:44:41,316 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138 after 0ms 2024-12-04T09:44:41,320 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:41,322 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138 took 5ms 2024-12-04T09:44:41,327 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733305481138.temp 2024-12-04T09:44:41,329 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003001-wal.1733305481138.temp 2024-12-04T09:44:41,377 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138 so closing down 2024-12-04T09:44:41,377 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:41,377 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:41,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741869_1046 (size=200484) 2024-12-04T09:44:41,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741869_1046 (size=200484) 2024-12-04T09:44:41,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741869_1046 (size=200484) 2024-12-04T09:44:41,382 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003001-wal.1733305481138.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-12-04T09:44:41,384 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003001-wal.1733305481138.temp to hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000006000 2024-12-04T09:44:41,384 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-04T09:44:41,385 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138, journal: Splitting hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138, size=195.8 K (200484bytes) at 1733305481316Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003001-wal.1733305481138.temp at 1733305481329 (+13 ms)Split 1024 edits, skipped 0 edits. at 1733305481343 (+14 ms)Split 2048 edits, skipped 0 edits. at 1733305481359 (+16 ms)Finishing writing output for hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138 so closing down at 1733305481377 (+18 ms)3 split writer threads finished at 1733305481377Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003001-wal.1733305481138.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1733305481382 (+5 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003001-wal.1733305481138.temp to hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000006000 at 1733305481384 (+2 ms)Processed 3000 edits across 1 Regions in 62 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733305481384 2024-12-04T09:44:41,387 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481138 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305481138 2024-12-04T09:44:41,388 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000006000 2024-12-04T09:44:41,388 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:41,391 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/test2727-manual,16010,1733305480802, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:41,411 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481392, exclude list is [], retry=0 2024-12-04T09:44:41,415 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:41,415 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:41,416 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:41,419 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733305480802/wal.1733305481392 2024-12-04T09:44:41,420 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:44:41,420 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0854c8f4d5538dd9a2ef75512d623e64, NAME => 'test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:41,420 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:41,420 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,421 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,423 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,424 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0854c8f4d5538dd9a2ef75512d623e64 columnFamilyName a 2024-12-04T09:44:41,424 DEBUG [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:41,425 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(327): Store=0854c8f4d5538dd9a2ef75512d623e64/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:41,425 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,427 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0854c8f4d5538dd9a2ef75512d623e64 columnFamilyName b 2024-12-04T09:44:41,427 DEBUG [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:41,428 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(327): Store=0854c8f4d5538dd9a2ef75512d623e64/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:41,428 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,430 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0854c8f4d5538dd9a2ef75512d623e64 columnFamilyName c 2024-12-04T09:44:41,430 DEBUG [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:41,431 INFO [StoreOpener-0854c8f4d5538dd9a2ef75512d623e64-1 {}] regionserver.HStore(327): Store=0854c8f4d5538dd9a2ef75512d623e64/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:41,431 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,432 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,435 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,436 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003000 2024-12-04T09:44:41,440 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:41,481 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003000 2024-12-04T09:44:41,482 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000006000 2024-12-04T09:44:41,486 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:41,539 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000006000 2024-12-04T09:44:41,539 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0854c8f4d5538dd9a2ef75512d623e64 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-04T09:44:41,563 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/a/0a91f1e0acbc430093b100fe2b94aefd is 41, key is test2727/a:100/1733305481162/Put/seqid=0 2024-12-04T09:44:41,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741871_1048 (size=84227) 2024-12-04T09:44:41,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741871_1048 (size=84227) 2024-12-04T09:44:41,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741871_1048 (size=84227) 2024-12-04T09:44:41,571 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/a/0a91f1e0acbc430093b100fe2b94aefd 2024-12-04T09:44:41,601 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/b/b0c5d8a864134c88beb2b2b10a224a7f is 41, key is test2727/b:100/1733305481207/Put/seqid=0 2024-12-04T09:44:41,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741872_1049 (size=84609) 2024-12-04T09:44:41,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741872_1049 (size=84609) 2024-12-04T09:44:41,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741872_1049 (size=84609) 2024-12-04T09:44:41,607 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/b/b0c5d8a864134c88beb2b2b10a224a7f 2024-12-04T09:44:41,632 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/c/164b364d465d4360b2cd6dd0f5d6d427 is 41, key is test2727/c:100/1733305481245/Put/seqid=0 2024-12-04T09:44:41,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741873_1050 (size=84609) 2024-12-04T09:44:41,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741873_1050 (size=84609) 2024-12-04T09:44:41,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741873_1050 (size=84609) 2024-12-04T09:44:41,639 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/c/164b364d465d4360b2cd6dd0f5d6d427 2024-12-04T09:44:41,647 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/a/0a91f1e0acbc430093b100fe2b94aefd as hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/a/0a91f1e0acbc430093b100fe2b94aefd 2024-12-04T09:44:41,655 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/a/0a91f1e0acbc430093b100fe2b94aefd, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-04T09:44:41,656 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/b/b0c5d8a864134c88beb2b2b10a224a7f as hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/b/b0c5d8a864134c88beb2b2b10a224a7f 2024-12-04T09:44:41,664 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/b/b0c5d8a864134c88beb2b2b10a224a7f, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T09:44:41,665 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/.tmp/c/164b364d465d4360b2cd6dd0f5d6d427 as hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/c/164b364d465d4360b2cd6dd0f5d6d427 2024-12-04T09:44:41,673 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/c/164b364d465d4360b2cd6dd0f5d6d427, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T09:44:41,673 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0854c8f4d5538dd9a2ef75512d623e64 in 134ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-04T09:44:41,674 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000003000 2024-12-04T09:44:41,675 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/0000000000000006000 2024-12-04T09:44:41,677 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,677 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,678 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:41,680 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0854c8f4d5538dd9a2ef75512d623e64 2024-12-04T09:44:41,683 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/test2727/0854c8f4d5538dd9a2ef75512d623e64/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-04T09:44:41,684 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0854c8f4d5538dd9a2ef75512d623e64; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68945748, jitterRate=0.027371704578399658}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:41,685 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0854c8f4d5538dd9a2ef75512d623e64: Writing region info on filesystem at 1733305481421Initializing all the Stores at 1733305481422 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305481422Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305481422Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305481422Obtaining lock to block concurrent updates at 1733305481539 (+117 ms)Preparing flush snapshotting stores in 0854c8f4d5538dd9a2ef75512d623e64 at 1733305481539Finished memstore snapshotting test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733305481539Flushing stores of test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. at 1733305481539Flushing 0854c8f4d5538dd9a2ef75512d623e64/a: creating writer at 1733305481539Flushing 0854c8f4d5538dd9a2ef75512d623e64/a: appending metadata at 1733305481562 (+23 ms)Flushing 0854c8f4d5538dd9a2ef75512d623e64/a: closing flushed file at 1733305481563 (+1 ms)Flushing 0854c8f4d5538dd9a2ef75512d623e64/b: creating writer at 1733305481580 (+17 ms)Flushing 0854c8f4d5538dd9a2ef75512d623e64/b: appending metadata at 1733305481599 (+19 ms)Flushing 0854c8f4d5538dd9a2ef75512d623e64/b: closing flushed file at 1733305481599Flushing 0854c8f4d5538dd9a2ef75512d623e64/c: creating writer at 1733305481614 (+15 ms)Flushing 0854c8f4d5538dd9a2ef75512d623e64/c: appending metadata at 1733305481631 (+17 ms)Flushing 0854c8f4d5538dd9a2ef75512d623e64/c: closing flushed file at 1733305481631Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28f6e477: reopening flushed file at 1733305481646 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@418b7921: reopening flushed file at 1733305481655 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8b06cf3: reopening flushed file at 1733305481664 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0854c8f4d5538dd9a2ef75512d623e64 in 134ms, sequenceid=6000, compaction requested=false; wal=null at 1733305481673 (+9 ms)Cleaning up temporary data from old regions at 1733305481677 (+4 ms)Region opened successfully at 1733305481685 (+8 ms) 2024-12-04T09:44:41,686 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-04T09:44:41,686 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0854c8f4d5538dd9a2ef75512d623e64, disabling compactions & flushes 2024-12-04T09:44:41,686 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:41,686 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:41,686 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. after waiting 0 ms 2024-12-04T09:44:41,686 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:41,688 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733305480804.0854c8f4d5538dd9a2ef75512d623e64. 2024-12-04T09:44:41,688 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0854c8f4d5538dd9a2ef75512d623e64: Waiting for close lock at 1733305481686Disabling compacts and flushes for region at 1733305481686Disabling writes for close at 1733305481686Writing region close event to WAL at 1733305481687 (+1 ms)Closed at 1733305481687 2024-12-04T09:44:41,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741870_1047 (size=93) 2024-12-04T09:44:41,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741870_1047 (size=93) 2024-12-04T09:44:41,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741870_1047 (size=93) 2024-12-04T09:44:41,693 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:41,693 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733305481392) 2024-12-04T09:44:41,704 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=400 (was 398) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:34844 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:60354 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:44262 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 423), ProcessCount=11 (was 11), AvailableMemoryMB=10725 (was 10946) 2024-12-04T09:44:41,715 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=400, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=423, ProcessCount=11, AvailableMemoryMB=10724 2024-12-04T09:44:41,731 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:41,737 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:41,738 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733305481737 2024-12-04T09:44:41,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737 2024-12-04T09:44:41,747 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:41,748 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3df61d85043adde7d025807a36ab903f, NAME => 'testSequentialEditLogSeqNum,,1733305481731.3df61d85043adde7d025807a36ab903f.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:41,748 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733305481731.3df61d85043adde7d025807a36ab903f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:41,748 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,748 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,749 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f doesn't exist for region: 3df61d85043adde7d025807a36ab903f on table testSequentialEditLogSeqNum 2024-12-04T09:44:41,750 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 3df61d85043adde7d025807a36ab903f on table testSequentialEditLogSeqNum 2024-12-04T09:44:41,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741875_1052 (size=62) 2024-12-04T09:44:41,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741875_1052 (size=62) 2024-12-04T09:44:41,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741875_1052 (size=62) 2024-12-04T09:44:41,761 INFO [StoreOpener-3df61d85043adde7d025807a36ab903f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,762 INFO [StoreOpener-3df61d85043adde7d025807a36ab903f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3df61d85043adde7d025807a36ab903f columnFamilyName a 2024-12-04T09:44:41,762 DEBUG [StoreOpener-3df61d85043adde7d025807a36ab903f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:41,763 INFO [StoreOpener-3df61d85043adde7d025807a36ab903f-1 {}] regionserver.HStore(327): Store=3df61d85043adde7d025807a36ab903f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:41,763 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,764 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,764 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,764 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,764 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,766 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3df61d85043adde7d025807a36ab903f 2024-12-04T09:44:41,768 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:41,769 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3df61d85043adde7d025807a36ab903f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62035695, jitterRate=-0.07559610903263092}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:44:41,770 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3df61d85043adde7d025807a36ab903f: Writing region info on filesystem at 1733305481749Initializing all the Stores at 1733305481760 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305481760Cleaning up temporary data from old regions at 1733305481765 (+5 ms)Region opened successfully at 1733305481770 (+5 ms) 2024-12-04T09:44:41,782 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3df61d85043adde7d025807a36ab903f 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-04T09:44:41,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/.tmp/a/6046eed4f87d451eaa9d895e48375a4f is 81, key is testSequentialEditLogSeqNum/a:x0/1733305481770/Put/seqid=0 2024-12-04T09:44:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741876_1053 (size=5833) 2024-12-04T09:44:41,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741876_1053 (size=5833) 2024-12-04T09:44:41,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741876_1053 (size=5833) 2024-12-04T09:44:41,835 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/.tmp/a/6046eed4f87d451eaa9d895e48375a4f 2024-12-04T09:44:41,842 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/.tmp/a/6046eed4f87d451eaa9d895e48375a4f as hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/a/6046eed4f87d451eaa9d895e48375a4f 2024-12-04T09:44:41,850 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/a/6046eed4f87d451eaa9d895e48375a4f, entries=10, sequenceid=13, filesize=5.7 K 2024-12-04T09:44:41,851 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 3df61d85043adde7d025807a36ab903f in 69ms, sequenceid=13, compaction requested=false 2024-12-04T09:44:41,852 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3df61d85043adde7d025807a36ab903f: 2024-12-04T09:44:41,861 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T09:44:41,861 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T09:44:41,861 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T09:44:41,861 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T09:44:41,862 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T09:44:41,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741874_1051 (size=1616) 2024-12-04T09:44:41,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741874_1051 (size=1616) 2024-12-04T09:44:41,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741874_1051 (size=1616) 2024-12-04T09:44:41,880 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737, size=1.6 K (1616bytes) 2024-12-04T09:44:41,881 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737 2024-12-04T09:44:41,881 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737 after 0ms 2024-12-04T09:44:41,884 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:41,885 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737 took 5ms 2024-12-04T09:44:41,887 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737 so closing down 2024-12-04T09:44:41,887 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:41,888 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733305481737.temp 2024-12-04T09:44:41,891 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000003-wal.1733305481737.temp 2024-12-04T09:44:41,891 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:41,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741877_1054 (size=1175) 2024-12-04T09:44:41,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741877_1054 (size=1175) 2024-12-04T09:44:41,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741877_1054 (size=1175) 2024-12-04T09:44:41,905 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000003-wal.1733305481737.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:41,907 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000003-wal.1733305481737.temp to hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000020 2024-12-04T09:44:41,907 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-04T09:44:41,907 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737, journal: Splitting hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737, size=1.6 K (1616bytes) at 1733305481880Finishing writing output for hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737 so closing down at 1733305481887 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000003-wal.1733305481737.temp at 1733305481891 (+4 ms)3 split writer threads finished at 1733305481891Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000003-wal.1733305481737.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733305481905 (+14 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000003-wal.1733305481737.temp to hdfs://localhost:33391/hbase/data/default/testSequentialEditLogSeqNum/3df61d85043adde7d025807a36ab903f/recovered.edits/0000000000000000020 at 1733305481907 (+2 ms)Processed 17 edits across 1 Regions in 22 ms; skipped=2; WAL=hdfs://localhost:33391/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733305481730/wal.1733305481737, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733305481907 2024-12-04T09:44:41,922 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=405 (was 400) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:34844 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:60354 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 423), ProcessCount=11 (was 11), AvailableMemoryMB=10655 (was 10724) 2024-12-04T09:44:41,935 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=405, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=423, ProcessCount=11, AvailableMemoryMB=10654 2024-12-04T09:44:41,954 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:41,957 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:41,991 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:41,994 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-90123261, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-90123261, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:42,005 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-90123261/hregion-90123261.1733305481995, exclude list is [], retry=0 2024-12-04T09:44:42,008 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:42,008 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:42,008 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:42,010 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-90123261/hregion-90123261.1733305481995 2024-12-04T09:44:42,010 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:44:42,011 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d1edd18b48509f22f48e5bba7352f71b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:42,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741879_1056 (size=70) 2024-12-04T09:44:42,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741879_1056 (size=70) 2024-12-04T09:44:42,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741879_1056 (size=70) 2024-12-04T09:44:42,027 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:42,029 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,030 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName a 2024-12-04T09:44:42,030 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:42,031 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:42,031 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,032 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName b 2024-12-04T09:44:42,032 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:42,033 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:42,033 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,034 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName c 2024-12-04T09:44:42,034 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:42,035 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:42,035 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,036 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,037 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,038 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,038 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,039 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:42,040 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,043 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:42,043 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1edd18b48509f22f48e5bba7352f71b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68587989, jitterRate=0.022040680050849915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:42,044 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1edd18b48509f22f48e5bba7352f71b: Writing region info on filesystem at 1733305482028Initializing all the Stores at 1733305482028Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305482028Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305482029 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305482029Cleaning up temporary data from old regions at 1733305482038 (+9 ms)Region opened successfully at 1733305482044 (+6 ms) 2024-12-04T09:44:42,045 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d1edd18b48509f22f48e5bba7352f71b, disabling compactions & flushes 2024-12-04T09:44:42,045 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:42,045 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:42,045 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. after waiting 0 ms 2024-12-04T09:44:42,045 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:42,045 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:42,046 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d1edd18b48509f22f48e5bba7352f71b: Waiting for close lock at 1733305482045Disabling compacts and flushes for region at 1733305482045Disabling writes for close at 1733305482045Writing region close event to WAL at 1733305482045Closed at 1733305482045 2024-12-04T09:44:42,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741878_1055 (size=93) 2024-12-04T09:44:42,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741878_1055 (size=93) 2024-12-04T09:44:42,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741878_1055 (size=93) 2024-12-04T09:44:42,051 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:42,052 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-90123261:(num 1733305481995) 2024-12-04T09:44:42,052 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:42,054 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:42,073 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055, exclude list is [], retry=0 2024-12-04T09:44:42,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:42,076 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:42,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:42,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T09:44:42,077 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:42,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T09:44:42,079 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T09:44:42,079 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T09:44:42,079 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:42,080 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-04T09:44:42,080 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:42,081 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-04T09:44:42,081 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:42,082 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 2024-12-04T09:44:42,082 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:44:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d1edd18b48509f22f48e5bba7352f71b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:42,083 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,083 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,084 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,086 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName a 2024-12-04T09:44:42,086 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:42,086 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:42,087 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,088 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName b 2024-12-04T09:44:42,088 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:42,089 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:42,089 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,090 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName c 2024-12-04T09:44:42,090 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:42,090 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:42,090 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,091 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,092 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,093 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,093 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,094 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:42,095 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:42,096 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1edd18b48509f22f48e5bba7352f71b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69159680, jitterRate=0.030559539794921875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:42,097 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1edd18b48509f22f48e5bba7352f71b: Writing region info on filesystem at 1733305482083Initializing all the Stores at 1733305482084 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305482084Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305482084Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305482084Cleaning up temporary data from old regions at 1733305482093 (+9 ms)Region opened successfully at 1733305482097 (+4 ms) 2024-12-04T09:44:42,102 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733305482101/Put/seqid=0 2024-12-04T09:44:42,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741881_1058 (size=4826) 2024-12-04T09:44:42,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741881_1058 (size=4826) 2024-12-04T09:44:42,110 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33391/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in d1edd18b48509f22f48e5bba7352f71b/a 2024-12-04T09:44:42,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741881_1058 (size=4826) 2024-12-04T09:44:42,119 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-04T09:44:42,119 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T09:44:42,120 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d1edd18b48509f22f48e5bba7352f71b: 2024-12-04T09:44:42,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/a/6822549ec5d94b20b64b29ab52c1ece5_SeqId_3_ 2024-12-04T09:44:42,122 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33391/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d1edd18b48509f22f48e5bba7352f71b/a as hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/a/6822549ec5d94b20b64b29ab52c1ece5_SeqId_3_ - updating store file list. 2024-12-04T09:44:42,129 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6822549ec5d94b20b64b29ab52c1ece5_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T09:44:42,129 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/a/6822549ec5d94b20b64b29ab52c1ece5_SeqId_3_ into d1edd18b48509f22f48e5bba7352f71b/a 2024-12-04T09:44:42,129 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33391/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into d1edd18b48509f22f48e5bba7352f71b/a (new location: hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/a/6822549ec5d94b20b64b29ab52c1ece5_SeqId_3_) 2024-12-04T09:44:42,166 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055, size=0 (0bytes) 2024-12-04T09:44:42,167 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 might be still open, length is 0 2024-12-04T09:44:42,167 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 2024-12-04T09:44:42,167 WARN [IPC Server handler 4 on default port 33391 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-04T09:44:42,167 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 after 0ms 2024-12-04T09:44:42,815 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:34978 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34978 dst: /127.0.0.1:40851 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40851 remote=/127.0.0.1:34978]. Total timeout mills is 60000, 59320 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:42,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:44346 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33979:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44346 dst: /127.0.0.1:33979 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:42,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:60452 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36275:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60452 dst: /127.0.0.1:36275 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:42,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741880_1059 (size=434) 2024-12-04T09:44:42,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741880_1059 (size=434) 2024-12-04T09:44:46,169 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 after 4002ms 2024-12-04T09:44:46,177 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:46,177 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 took 4011ms 2024-12-04T09:44:46,181 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055; continuing. 2024-12-04T09:44:46,181 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 so closing down 2024-12-04T09:44:46,182 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:46,183 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733305482055.temp 2024-12-04T09:44:46,185 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005-wal.1733305482055.temp 2024-12-04T09:44:46,185 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:46,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741882_1060 (size=236) 2024-12-04T09:44:46,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741882_1060 (size=236) 2024-12-04T09:44:46,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741882_1060 (size=236) 2024-12-04T09:44:46,194 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005-wal.1733305482055.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:46,196 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005-wal.1733305482055.temp to hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005 2024-12-04T09:44:46,196 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T09:44:46,196 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055, journal: Splitting hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055, size=0 (0bytes) at 1733305482166Finishing writing output for hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 so closing down at 1733305486181 (+4015 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005-wal.1733305482055.temp at 1733305486185 (+4 ms)3 split writer threads finished at 1733305486185Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005-wal.1733305482055.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733305486194 (+9 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005-wal.1733305482055.temp to hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005 at 1733305486196 (+2 ms)Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055, size=0, length=0, corrupted=false, cancelled=false at 1733305486196 2024-12-04T09:44:46,197 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305482055 2024-12-04T09:44:46,198 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005 2024-12-04T09:44:46,198 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:46,200 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:46,211 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305486200, exclude list is [], retry=0 2024-12-04T09:44:46,213 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:46,213 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:46,214 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:46,215 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305486200 2024-12-04T09:44:46,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d1edd18b48509f22f48e5bba7352f71b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,217 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,218 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName a 2024-12-04T09:44:46,218 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:46,225 DEBUG [StoreFileOpener-d1edd18b48509f22f48e5bba7352f71b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6822549ec5d94b20b64b29ab52c1ece5_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T09:44:46,225 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/a/6822549ec5d94b20b64b29ab52c1ece5_SeqId_3_ 2024-12-04T09:44:46,226 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:46,226 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,227 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName b 2024-12-04T09:44:46,227 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:46,227 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:46,227 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,228 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d1edd18b48509f22f48e5bba7352f71b columnFamilyName c 2024-12-04T09:44:46,228 DEBUG [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:46,229 INFO [StoreOpener-d1edd18b48509f22f48e5bba7352f71b-1 {}] regionserver.HStore(327): Store=d1edd18b48509f22f48e5bba7352f71b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:46,229 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,229 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,231 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,232 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005 2024-12-04T09:44:46,234 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:46,235 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005 2024-12-04T09:44:46,235 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d1edd18b48509f22f48e5bba7352f71b 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-04T09:44:46,248 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/.tmp/a/9658afa6e0d548478958af0b216c6e38 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733305482134/Put/seqid=0 2024-12-04T09:44:46,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741884_1062 (size=5149) 2024-12-04T09:44:46,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741884_1062 (size=5149) 2024-12-04T09:44:46,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741884_1062 (size=5149) 2024-12-04T09:44:46,255 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/.tmp/a/9658afa6e0d548478958af0b216c6e38 2024-12-04T09:44:46,260 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/.tmp/a/9658afa6e0d548478958af0b216c6e38 as hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/a/9658afa6e0d548478958af0b216c6e38 2024-12-04T09:44:46,265 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/a/9658afa6e0d548478958af0b216c6e38, entries=1, sequenceid=5, filesize=5.0 K 2024-12-04T09:44:46,266 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d1edd18b48509f22f48e5bba7352f71b in 31ms, sequenceid=5, compaction requested=false; wal=null 2024-12-04T09:44:46,266 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/0000000000000000005 2024-12-04T09:44:46,268 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,268 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,268 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:46,270 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d1edd18b48509f22f48e5bba7352f71b 2024-12-04T09:44:46,272 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/d1edd18b48509f22f48e5bba7352f71b/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T09:44:46,273 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d1edd18b48509f22f48e5bba7352f71b; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67058035, jitterRate=-7.574111223220825E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:46,273 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d1edd18b48509f22f48e5bba7352f71b: Writing region info on filesystem at 1733305486216Initializing all the Stores at 1733305486217 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305486217Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305486217Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305486217Obtaining lock to block concurrent updates at 1733305486235 (+18 ms)Preparing flush snapshotting stores in d1edd18b48509f22f48e5bba7352f71b at 1733305486235Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733305486235Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. at 1733305486235Flushing d1edd18b48509f22f48e5bba7352f71b/a: creating writer at 1733305486235Flushing d1edd18b48509f22f48e5bba7352f71b/a: appending metadata at 1733305486248 (+13 ms)Flushing d1edd18b48509f22f48e5bba7352f71b/a: closing flushed file at 1733305486248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22f1bf3: reopening flushed file at 1733305486259 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for d1edd18b48509f22f48e5bba7352f71b in 31ms, sequenceid=5, compaction requested=false; wal=null at 1733305486266 (+7 ms)Cleaning up temporary data from old regions at 1733305486268 (+2 ms)Region opened successfully at 1733305486273 (+5 ms) 2024-12-04T09:44:46,278 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d1edd18b48509f22f48e5bba7352f71b, disabling compactions & flushes 2024-12-04T09:44:46,278 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:46,278 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:46,278 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. after waiting 0 ms 2024-12-04T09:44:46,278 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:46,279 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733305481955.d1edd18b48509f22f48e5bba7352f71b. 2024-12-04T09:44:46,279 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d1edd18b48509f22f48e5bba7352f71b: Waiting for close lock at 1733305486278Disabling compacts and flushes for region at 1733305486278Disabling writes for close at 1733305486278Writing region close event to WAL at 1733305486279 (+1 ms)Closed at 1733305486279 2024-12-04T09:44:46,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741883_1061 (size=93) 2024-12-04T09:44:46,282 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305486200 not finished, retry = 0 2024-12-04T09:44:46,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741883_1061 (size=93) 2024-12-04T09:44:46,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741883_1061 (size=93) 2024-12-04T09:44:46,386 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:46,386 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733305486200) 2024-12-04T09:44:46,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=407 (was 405) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434950674_22 at /127.0.0.1:44374 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434950674_22 at /127.0.0.1:35008 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33391 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1434950674_22 at /127.0.0.1:60482 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:33391 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=389 (was 423), ProcessCount=11 (was 11), AvailableMemoryMB=10662 (was 10654) - AvailableMemoryMB LEAK? - 2024-12-04T09:44:46,417 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=407, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=10661 2024-12-04T09:44:46,435 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:46,439 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T09:44:46,442 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 84486a41f81c,38097,1733305471718 2024-12-04T09:44:46,445 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1bba1bf9 2024-12-04T09:44:46,446 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T09:44:46,448 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T09:44:46,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T09:44:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T09:44:46,459 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T09:44:46,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-04T09:44:46,461 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:46,462 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T09:44:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T09:44:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741885_1063 (size=694) 2024-12-04T09:44:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741885_1063 (size=694) 2024-12-04T09:44:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741885_1063 (size=694) 2024-12-04T09:44:46,476 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 56ae12cc64f9a720b1c872193297ac28, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095 2024-12-04T09:44:46,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741886_1064 (size=77) 2024-12-04T09:44:46,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741886_1064 (size=77) 2024-12-04T09:44:46,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741886_1064 (size=77) 2024-12-04T09:44:46,486 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:46,486 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 56ae12cc64f9a720b1c872193297ac28, disabling compactions & flushes 2024-12-04T09:44:46,487 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:46,487 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:46,487 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. after waiting 0 ms 2024-12-04T09:44:46,487 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:46,487 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:46,487 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 56ae12cc64f9a720b1c872193297ac28: Waiting for close lock at 1733305486486Disabling compacts and flushes for region at 1733305486486Disabling writes for close at 1733305486487 (+1 ms)Writing region close event to WAL at 1733305486487Closed at 1733305486487 2024-12-04T09:44:46,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T09:44:46,495 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733305486489"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733305486489"}]},"ts":"1733305486489"} 2024-12-04T09:44:46,498 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T09:44:46,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T09:44:46,503 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733305486500"}]},"ts":"1733305486500"} 2024-12-04T09:44:46,506 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-04T09:44:46,507 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {84486a41f81c=0} racks are {/default-rack=0} 2024-12-04T09:44:46,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T09:44:46,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T09:44:46,508 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T09:44:46,509 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T09:44:46,509 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T09:44:46,509 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T09:44:46,509 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T09:44:46,509 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T09:44:46,509 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T09:44:46,509 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T09:44:46,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN}] 2024-12-04T09:44:46,513 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN 2024-12-04T09:44:46,514 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN; state=OFFLINE, location=84486a41f81c,38215,1733305472546; forceNewPlan=false, retain=false 2024-12-04T09:44:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T09:44:46,667 INFO [84486a41f81c:38097 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T09:44:46,668 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPENING, regionLocation=84486a41f81c,38215,1733305472546 2024-12-04T09:44:46,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN because future has completed 2024-12-04T09:44:46,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546}] 2024-12-04T09:44:46,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T09:44:46,829 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T09:44:46,833 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T09:44:46,841 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:46,842 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 56ae12cc64f9a720b1c872193297ac28, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:46,842 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,842 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:46,842 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,842 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,844 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,845 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf1 2024-12-04T09:44:46,846 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:46,846 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:46,846 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,847 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf2 2024-12-04T09:44:46,847 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:46,848 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:46,848 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,849 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,849 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,850 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,850 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,851 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T09:44:46,853 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,855 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:46,856 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 56ae12cc64f9a720b1c872193297ac28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61094578, jitterRate=-0.08961984515190125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T09:44:46,856 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:46,857 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 56ae12cc64f9a720b1c872193297ac28: Running coprocessor pre-open hook at 1733305486843Writing region info on filesystem at 1733305486843Initializing all the Stores at 1733305486844 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305486844Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305486844Cleaning up temporary data from old regions at 1733305486850 (+6 ms)Running coprocessor post-open hooks at 1733305486856 (+6 ms)Region opened successfully at 1733305486857 (+1 ms) 2024-12-04T09:44:46,859 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., pid=6, masterSystemTime=1733305486828 2024-12-04T09:44:46,862 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:46,862 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:46,862 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPEN, openSeqNum=2, regionLocation=84486a41f81c,38215,1733305472546 2024-12-04T09:44:46,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546 because future has completed 2024-12-04T09:44:46,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T09:44:46,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546 in 194 msec 2024-12-04T09:44:46,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T09:44:46,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN in 360 msec 2024-12-04T09:44:46,875 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T09:44:46,875 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733305486875"}]},"ts":"1733305486875"} 2024-12-04T09:44:46,877 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-04T09:44:46,879 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T09:44:46,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 426 msec 2024-12-04T09:44:47,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T09:44:47,096 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-04T09:44:47,096 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-04T09:44:47,100 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T09:44:47,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-04T09:44:47,108 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T09:44:47,109 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-04T09:44:47,122 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=2] 2024-12-04T09:44:47,123 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T09:44:47,125 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T09:44:47,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=56ae12cc64f9a720b1c872193297ac28, source=84486a41f81c,38215,1733305472546, destination=84486a41f81c,35147,1733305472454, warming up region on 84486a41f81c,35147,1733305472454 2024-12-04T09:44:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T09:44:47,143 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=56ae12cc64f9a720b1c872193297ac28, source=84486a41f81c,38215,1733305472546, destination=84486a41f81c,35147,1733305472454, running balancer 2024-12-04T09:44:47,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57251, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T09:44:47,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE 2024-12-04T09:44:47,145 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE 2024-12-04T09:44:47,147 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=CLOSING, regionLocation=84486a41f81c,38215,1733305472546 2024-12-04T09:44:47,149 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38097 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=84486a41f81c,38215,1733305472546, table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-04T09:44:47,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(7855): Warmup {ENCODED => 56ae12cc64f9a720b1c872193297ac28, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:47,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:47,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE because future has completed 2024-12-04T09:44:47,150 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,151 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T09:44:47,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546}] 2024-12-04T09:44:47,152 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf1 2024-12-04T09:44:47,152 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:47,152 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:47,152 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,153 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf2 2024-12-04T09:44:47,153 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:47,154 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(1722): Closing 56ae12cc64f9a720b1c872193297ac28, disabling compactions & flushes 2024-12-04T09:44:47,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. after waiting 0 ms 2024-12-04T09:44:47,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] regionserver.HRegion(1676): Region close journal for 56ae12cc64f9a720b1c872193297ac28: Waiting for close lock at 1733305487154Disabling compacts and flushes for region at 1733305487154Disabling writes for close at 1733305487155 (+1 ms)Writing region close event to WAL at 1733305487155Closed at 1733305487155 2024-12-04T09:44:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-04T09:44:47,311 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,311 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T09:44:47,312 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 56ae12cc64f9a720b1c872193297ac28, disabling compactions & flushes 2024-12-04T09:44:47,312 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,313 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,313 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. after waiting 0 ms 2024-12-04T09:44:47,313 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,313 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 56ae12cc64f9a720b1c872193297ac28 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-04T09:44:47,335 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf1/d73a71c494a74159afdd17fc9749ab8b is 35, key is r1/cf1:q/1733305487126/Put/seqid=0 2024-12-04T09:44:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741887_1065 (size=4783) 2024-12-04T09:44:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741887_1065 (size=4783) 2024-12-04T09:44:47,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741887_1065 (size=4783) 2024-12-04T09:44:47,343 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf1/d73a71c494a74159afdd17fc9749ab8b 2024-12-04T09:44:47,350 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf1/d73a71c494a74159afdd17fc9749ab8b as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b 2024-12-04T09:44:47,358 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b, entries=1, sequenceid=5, filesize=4.7 K 2024-12-04T09:44:47,359 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 56ae12cc64f9a720b1c872193297ac28 in 46ms, sequenceid=5, compaction requested=false 2024-12-04T09:44:47,359 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-04T09:44:47,365 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T09:44:47,367 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,367 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 56ae12cc64f9a720b1c872193297ac28: Waiting for close lock at 1733305487312Running coprocessor pre-close hooks at 1733305487312Disabling compacts and flushes for region at 1733305487312Disabling writes for close at 1733305487313 (+1 ms)Obtaining lock to block concurrent updates at 1733305487313Preparing flush snapshotting stores in 56ae12cc64f9a720b1c872193297ac28 at 1733305487313Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733305487314 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. at 1733305487315 (+1 ms)Flushing 56ae12cc64f9a720b1c872193297ac28/cf1: creating writer at 1733305487315Flushing 56ae12cc64f9a720b1c872193297ac28/cf1: appending metadata at 1733305487335 (+20 ms)Flushing 56ae12cc64f9a720b1c872193297ac28/cf1: closing flushed file at 1733305487335Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40528dcf: reopening flushed file at 1733305487349 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 56ae12cc64f9a720b1c872193297ac28 in 46ms, sequenceid=5, compaction requested=false at 1733305487359 (+10 ms)Writing region close event to WAL at 1733305487361 (+2 ms)Running coprocessor post-close hooks at 1733305487365 (+4 ms)Closed at 1733305487367 (+2 ms) 2024-12-04T09:44:47,368 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 56ae12cc64f9a720b1c872193297ac28 move to 84486a41f81c,35147,1733305472454 record at close sequenceid=5 2024-12-04T09:44:47,371 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,371 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=CLOSED 2024-12-04T09:44:47,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546 because future has completed 2024-12-04T09:44:47,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T09:44:47,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546 in 225 msec 2024-12-04T09:44:47,381 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE; state=CLOSED, location=84486a41f81c,35147,1733305472454; forceNewPlan=false, retain=false 2024-12-04T09:44:47,531 INFO [84486a41f81c:38097 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T09:44:47,532 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPENING, regionLocation=84486a41f81c,35147,1733305472454 2024-12-04T09:44:47,535 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE because future has completed 2024-12-04T09:44:47,535 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454}] 2024-12-04T09:44:47,693 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,693 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 56ae12cc64f9a720b1c872193297ac28, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:47,694 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,694 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:47,694 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,694 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,696 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,697 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf1 2024-12-04T09:44:47,697 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:47,705 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b 2024-12-04T09:44:47,705 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:47,705 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,706 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf2 2024-12-04T09:44:47,706 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:47,707 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:47,707 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,708 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,709 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,710 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,710 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,711 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T09:44:47,713 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,714 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 56ae12cc64f9a720b1c872193297ac28; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59996213, jitterRate=-0.10598675906658173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T09:44:47,714 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:47,715 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 56ae12cc64f9a720b1c872193297ac28: Running coprocessor pre-open hook at 1733305487694Writing region info on filesystem at 1733305487694Initializing all the Stores at 1733305487695 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305487695Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305487695Cleaning up temporary data from old regions at 1733305487710 (+15 ms)Running coprocessor post-open hooks at 1733305487714 (+4 ms)Region opened successfully at 1733305487715 (+1 ms) 2024-12-04T09:44:47,716 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., pid=9, masterSystemTime=1733305487689 2024-12-04T09:44:47,718 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,719 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,719 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPEN, openSeqNum=9, regionLocation=84486a41f81c,35147,1733305472454 2024-12-04T09:44:47,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454 because future has completed 2024-12-04T09:44:47,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-04T09:44:47,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454 in 188 msec 2024-12-04T09:44:47,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE in 584 msec 2024-12-04T09:44:47,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T09:44:47,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T09:44:47,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:32972 deadline: 1733305547756, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=35147 startCode=1733305472454. As of locationSeqNum=5. 2024-12-04T09:44:47,785 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=35147 startCode=1733305472454. As of locationSeqNum=5. 2024-12-04T09:44:47,786 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=35147 startCode=1733305472454. As of locationSeqNum=5. 2024-12-04T09:44:47,786 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,35147,1733305472454, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=35147 startCode=1733305472454. As of locationSeqNum=5. 2024-12-04T09:44:47,893 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T09:44:47,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T09:44:47,907 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56ae12cc64f9a720b1c872193297ac28 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-04T09:44:47,922 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf1/53b73dede3ee4078b106c41ada28bbc5 is 29, key is r1/cf1:/1733305487897/DeleteFamily/seqid=0 2024-12-04T09:44:47,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741888_1066 (size=4906) 2024-12-04T09:44:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741888_1066 (size=4906) 2024-12-04T09:44:47,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741888_1066 (size=4906) 2024-12-04T09:44:47,934 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf1/53b73dede3ee4078b106c41ada28bbc5 2024-12-04T09:44:47,940 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 53b73dede3ee4078b106c41ada28bbc5 2024-12-04T09:44:47,954 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf2/bed239705e4e4cccacb387b37c7c4604 is 29, key is r1/cf2:/1733305487897/DeleteFamily/seqid=0 2024-12-04T09:44:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741889_1067 (size=4906) 2024-12-04T09:44:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741889_1067 (size=4906) 2024-12-04T09:44:47,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741889_1067 (size=4906) 2024-12-04T09:44:47,961 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf2/bed239705e4e4cccacb387b37c7c4604 2024-12-04T09:44:47,969 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bed239705e4e4cccacb387b37c7c4604 2024-12-04T09:44:47,970 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf1/53b73dede3ee4078b106c41ada28bbc5 as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/53b73dede3ee4078b106c41ada28bbc5 2024-12-04T09:44:47,977 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 53b73dede3ee4078b106c41ada28bbc5 2024-12-04T09:44:47,977 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/53b73dede3ee4078b106c41ada28bbc5, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T09:44:47,979 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf2/bed239705e4e4cccacb387b37c7c4604 as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/bed239705e4e4cccacb387b37c7c4604 2024-12-04T09:44:47,985 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bed239705e4e4cccacb387b37c7c4604 2024-12-04T09:44:47,985 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/bed239705e4e4cccacb387b37c7c4604, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T09:44:47,986 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 56ae12cc64f9a720b1c872193297ac28 in 79ms, sequenceid=12, compaction requested=false 2024-12-04T09:44:47,986 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 56ae12cc64f9a720b1c872193297ac28: 2024-12-04T09:44:47,989 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-04T09:44:47,990 DEBUG [Time-limited test {}] regionserver.HStore(1541): 56ae12cc64f9a720b1c872193297ac28/cf1 is initiating major compaction (all files) 2024-12-04T09:44:47,990 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T09:44:47,990 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:47,991 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 56ae12cc64f9a720b1c872193297ac28/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:47,991 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b, hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/53b73dede3ee4078b106c41ada28bbc5] into tmpdir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp, totalSize=9.5 K 2024-12-04T09:44:47,992 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d73a71c494a74159afdd17fc9749ab8b, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733305487126 2024-12-04T09:44:47,992 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 53b73dede3ee4078b106c41ada28bbc5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T09:44:48,003 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 56ae12cc64f9a720b1c872193297ac28#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T09:44:48,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741890_1068 (size=4626) 2024-12-04T09:44:48,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741890_1068 (size=4626) 2024-12-04T09:44:48,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741890_1068 (size=4626) 2024-12-04T09:44:48,019 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf1/8b776c96d0d0458a9afea98c13b9a9ac as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/8b776c96d0d0458a9afea98c13b9a9ac 2024-12-04T09:44:48,034 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 56ae12cc64f9a720b1c872193297ac28/cf1 of 56ae12cc64f9a720b1c872193297ac28 into 8b776c96d0d0458a9afea98c13b9a9ac(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T09:44:48,034 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 56ae12cc64f9a720b1c872193297ac28: 2024-12-04T09:44:48,034 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T09:44:48,034 DEBUG [Time-limited test {}] regionserver.HStore(1541): 56ae12cc64f9a720b1c872193297ac28/cf2 is initiating major compaction (all files) 2024-12-04T09:44:48,035 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T09:44:48,035 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T09:44:48,035 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 56ae12cc64f9a720b1c872193297ac28/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,035 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/bed239705e4e4cccacb387b37c7c4604] into tmpdir=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp, totalSize=4.8 K 2024-12-04T09:44:48,036 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bed239705e4e4cccacb387b37c7c4604, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T09:44:48,041 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 56ae12cc64f9a720b1c872193297ac28#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T09:44:48,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741891_1069 (size=4592) 2024-12-04T09:44:48,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741891_1069 (size=4592) 2024-12-04T09:44:48,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741891_1069 (size=4592) 2024-12-04T09:44:48,055 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/.tmp/cf2/52c1af6087ff4d93a5aa17f28812c17d as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/52c1af6087ff4d93a5aa17f28812c17d 2024-12-04T09:44:48,062 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 56ae12cc64f9a720b1c872193297ac28/cf2 of 56ae12cc64f9a720b1c872193297ac28 into 52c1af6087ff4d93a5aa17f28812c17d(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T09:44:48,062 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 56ae12cc64f9a720b1c872193297ac28: 2024-12-04T09:44:48,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=56ae12cc64f9a720b1c872193297ac28, source=84486a41f81c,35147,1733305472454, destination=84486a41f81c,38215,1733305472546, warming up region on 84486a41f81c,38215,1733305472546 2024-12-04T09:44:48,066 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=56ae12cc64f9a720b1c872193297ac28, source=84486a41f81c,35147,1733305472454, destination=84486a41f81c,38215,1733305472546, running balancer 2024-12-04T09:44:48,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE 2024-12-04T09:44:48,068 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE 2024-12-04T09:44:48,070 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=CLOSING, regionLocation=84486a41f81c,35147,1733305472454 2024-12-04T09:44:48,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(7855): Warmup {ENCODED => 56ae12cc64f9a720b1c872193297ac28, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:48,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:48,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE because future has completed 2024-12-04T09:44:48,073 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T09:44:48,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454}] 2024-12-04T09:44:48,073 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,075 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf1 2024-12-04T09:44:48,075 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:48,083 INFO [StoreFileOpener-56ae12cc64f9a720b1c872193297ac28-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 53b73dede3ee4078b106c41ada28bbc5 2024-12-04T09:44:48,083 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/53b73dede3ee4078b106c41ada28bbc5 2024-12-04T09:44:48,089 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/8b776c96d0d0458a9afea98c13b9a9ac 2024-12-04T09:44:48,095 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b 2024-12-04T09:44:48,095 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:48,095 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,097 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf2 2024-12-04T09:44:48,097 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:48,104 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/52c1af6087ff4d93a5aa17f28812c17d 2024-12-04T09:44:48,111 INFO [StoreFileOpener-56ae12cc64f9a720b1c872193297ac28-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bed239705e4e4cccacb387b37c7c4604 2024-12-04T09:44:48,111 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/bed239705e4e4cccacb387b37c7c4604 2024-12-04T09:44:48,111 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(1722): Closing 56ae12cc64f9a720b1c872193297ac28, disabling compactions & flushes 2024-12-04T09:44:48,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. after waiting 0 ms 2024-12-04T09:44:48,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38215 {}] regionserver.HRegion(1676): Region close journal for 56ae12cc64f9a720b1c872193297ac28: Waiting for close lock at 1733305488111Disabling compacts and flushes for region at 1733305488111Disabling writes for close at 1733305488111Writing region close event to WAL at 1733305488113 (+2 ms)Closed at 1733305488113 2024-12-04T09:44:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-04T09:44:48,228 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,228 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T09:44:48,228 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 56ae12cc64f9a720b1c872193297ac28, disabling compactions & flushes 2024-12-04T09:44:48,228 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,228 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,228 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. after waiting 0 ms 2024-12-04T09:44:48,228 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,229 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b, hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/53b73dede3ee4078b106c41ada28bbc5] to archive 2024-12-04T09:44:48,233 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T09:44:48,237 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b to hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/d73a71c494a74159afdd17fc9749ab8b 2024-12-04T09:44:48,239 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/53b73dede3ee4078b106c41ada28bbc5 to hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/53b73dede3ee4078b106c41ada28bbc5 2024-12-04T09:44:48,255 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/bed239705e4e4cccacb387b37c7c4604] to archive 2024-12-04T09:44:48,256 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T09:44:48,258 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/bed239705e4e4cccacb387b37c7c4604 to hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/bed239705e4e4cccacb387b37c7c4604 2024-12-04T09:44:48,264 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-04T09:44:48,265 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,265 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 56ae12cc64f9a720b1c872193297ac28: Waiting for close lock at 1733305488228Running coprocessor pre-close hooks at 1733305488228Disabling compacts and flushes for region at 1733305488228Disabling writes for close at 1733305488228Writing region close event to WAL at 1733305488260 (+32 ms)Running coprocessor post-close hooks at 1733305488265 (+5 ms)Closed at 1733305488265 2024-12-04T09:44:48,265 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 56ae12cc64f9a720b1c872193297ac28 move to 84486a41f81c,38215,1733305472546 record at close sequenceid=12 2024-12-04T09:44:48,268 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,268 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=CLOSED 2024-12-04T09:44:48,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454 because future has completed 2024-12-04T09:44:48,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T09:44:48,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454 in 200 msec 2024-12-04T09:44:48,278 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE; state=CLOSED, location=84486a41f81c,38215,1733305472546; forceNewPlan=false, retain=false 2024-12-04T09:44:48,429 INFO [84486a41f81c:38097 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T09:44:48,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPENING, regionLocation=84486a41f81c,38215,1733305472546 2024-12-04T09:44:48,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE because future has completed 2024-12-04T09:44:48,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546}] 2024-12-04T09:44:48,590 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,590 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 56ae12cc64f9a720b1c872193297ac28, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:48,591 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,591 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:48,591 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,591 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,592 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,593 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf1 2024-12-04T09:44:48,593 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:48,599 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/8b776c96d0d0458a9afea98c13b9a9ac 2024-12-04T09:44:48,600 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:48,600 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,601 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf2 2024-12-04T09:44:48,601 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:48,607 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/52c1af6087ff4d93a5aa17f28812c17d 2024-12-04T09:44:48,607 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:48,607 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,608 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,609 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,610 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,610 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,611 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T09:44:48,612 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,613 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 56ae12cc64f9a720b1c872193297ac28; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65215207, jitterRate=-0.02821768820285797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T09:44:48,613 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,614 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 56ae12cc64f9a720b1c872193297ac28: Running coprocessor pre-open hook at 1733305488591Writing region info on filesystem at 1733305488591Initializing all the Stores at 1733305488592 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305488592Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305488592Cleaning up temporary data from old regions at 1733305488610 (+18 ms)Running coprocessor post-open hooks at 1733305488613 (+3 ms)Region opened successfully at 1733305488614 (+1 ms) 2024-12-04T09:44:48,615 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., pid=12, masterSystemTime=1733305488585 2024-12-04T09:44:48,617 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,618 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,618 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPEN, openSeqNum=18, regionLocation=84486a41f81c,38215,1733305472546 2024-12-04T09:44:48,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546 because future has completed 2024-12-04T09:44:48,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-04T09:44:48,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,38215,1733305472546 in 190 msec 2024-12-04T09:44:48,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, REOPEN/MOVE in 558 msec 2024-12-04T09:44:48,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T09:44:48,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T09:44:48,678 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 84486a41f81c,38215,1733305472546: testing ***** 2024-12-04T09:44:48,678 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-04T09:44:48,681 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-04T09:44:48,683 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-04T09:44:48,687 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-04T09:44:48,689 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-04T09:44:48,700 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 346082304 }, "NonHeapMemoryUsage": { "committed": 170459136, "init": 7667712, "max": -1, "used": 167891240 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "84486a41f81c", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "84486a41f81c", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:64397", "tag.serverName": "84486a41f81c,35147,1733305472454", "tag.clusterId": "e0ed37ab-3fa5-43a0-81c4-b1b6ab95a61c", "tag.Context": "regionserver", "tag.Hostname": "84486a41f81c", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733305472454, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-04T09:44:48,703 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38097 {}] master.MasterRpcServices(700): 84486a41f81c,38215,1733305472546 reported a fatal error: ***** ABORTING region server 84486a41f81c,38215,1733305472546: testing ***** 2024-12-04T09:44:48,706 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,38215,1733305472546' ***** 2024-12-04T09:44:48,706 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-04T09:44:48,706 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T09:44:48,707 INFO [RS:1;84486a41f81c:38215 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-04T09:44:48,707 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T09:44:48,707 INFO [RS:1;84486a41f81c:38215 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-04T09:44:48,707 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(3091): Received CLOSE for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,707 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(956): aborting server 84486a41f81c,38215,1733305472546 2024-12-04T09:44:48,707 INFO [RS:1;84486a41f81c:38215 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T09:44:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35147 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:55094 deadline: 1733305548707, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=38215 startCode=1733305472546. As of locationSeqNum=12. 2024-12-04T09:44:48,708 INFO [RS:1;84486a41f81c:38215 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;84486a41f81c:38215. 2024-12-04T09:44:48,708 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 56ae12cc64f9a720b1c872193297ac28, disabling compactions & flushes 2024-12-04T09:44:48,708 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,708 DEBUG [RS:1;84486a41f81c:38215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T09:44:48,708 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,708 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. after waiting 0 ms 2024-12-04T09:44:48,708 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,708 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,35147,1733305472454, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,35147,1733305472454, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=38215 startCode=1733305472546. As of locationSeqNum=12. 2024-12-04T09:44:48,708 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,35147,1733305472454, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=38215 startCode=1733305472546. As of locationSeqNum=12. 2024-12-04T09:44:48,708 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,35147,1733305472454, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=84486a41f81c port=38215 startCode=1733305472546. As of locationSeqNum=12. 2024-12-04T09:44:48,708 DEBUG [RS:1;84486a41f81c:38215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:44:48,709 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T09:44:48,709 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1325): Online Regions={56ae12cc64f9a720b1c872193297ac28=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.} 2024-12-04T09:44:48,709 DEBUG [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1351): Waiting on 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:48,711 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,711 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 56ae12cc64f9a720b1c872193297ac28: Waiting for close lock at 1733305488707Running coprocessor pre-close hooks at 1733305488708 (+1 ms)Disabling compacts and flushes for region at 1733305488708Disabling writes for close at 1733305488708Writing region close event to WAL at 1733305488711 (+3 ms)Running coprocessor post-close hooks at 1733305488711Closed at 1733305488711 2024-12-04T09:44:48,711 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:48,781 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T09:44:48,815 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 84486a41f81c,38215,1733305472546 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:48,816 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 84486a41f81c,38215,1733305472546 aborting 2024-12-04T09:44:48,816 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 84486a41f81c,38215,1733305472546 aborting 2024-12-04T09:44:48,817 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=12 from cache 2024-12-04T09:44:48,864 INFO [regionserver/84486a41f81c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-04T09:44:48,864 INFO [regionserver/84486a41f81c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-04T09:44:48,909 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,38215,1733305472546; all regions closed. 2024-12-04T09:44:48,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741835_1011 (size=1677) 2024-12-04T09:44:48,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741835_1011 (size=1677) 2024-12-04T09:44:48,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741835_1011 (size=1677) 2024-12-04T09:44:48,918 DEBUG [RS:1;84486a41f81c:38215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:44:48,918 INFO [RS:1;84486a41f81c:38215 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T09:44:48,918 INFO [RS:1;84486a41f81c:38215 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T09:44:48,919 INFO [RS:1;84486a41f81c:38215 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T09:44:48,919 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T09:44:48,919 INFO [RS:1;84486a41f81c:38215 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T09:44:48,920 INFO [RS:1;84486a41f81c:38215 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T09:44:48,920 INFO [RS:1;84486a41f81c:38215 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T09:44:48,920 INFO [RS:1;84486a41f81c:38215 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T09:44:48,920 INFO [RS:1;84486a41f81c:38215 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38215 2024-12-04T09:44:49,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,38215,1733305472546 2024-12-04T09:44:49,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T09:44:49,009 INFO [RS:1;84486a41f81c:38215 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T09:44:49,018 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,38215,1733305472546] 2024-12-04T09:44:49,025 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,38215,1733305472546 already deleted, retry=false 2024-12-04T09:44:49,026 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 84486a41f81c,38215,1733305472546 on 84486a41f81c,38097,1733305471718 2024-12-04T09:44:49,026 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18] 2024-12-04T09:44:49,027 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 84486a41f81c:38215 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 84486a41f81c/172.17.0.2:38215 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:49,028 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18, error=java.net.ConnectException: Call to address=84486a41f81c:38215 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 84486a41f81c/172.17.0.2:38215 2024-12-04T09:44:49,028 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-04T09:44:49,028 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18 from cache 2024-12-04T09:44:49,028 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 84486a41f81c:38215 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 84486a41f81c/172.17.0.2:38215 2024-12-04T09:44:49,031 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 84486a41f81c,38215,1733305472546, splitWal=true, meta=false 2024-12-04T09:44:49,034 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 84486a41f81c,38215,1733305472546 (carryingMeta=false) 84486a41f81c,38215,1733305472546/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@36f125fd[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-04T09:44:49,034 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 84486a41f81c,38215,1733305472546, splitWal=true, meta=false 2024-12-04T09:44:49,035 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 84486a41f81c,38215,1733305472546 had 1 regions 2024-12-04T09:44:49,037 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 84486a41f81c,38215,1733305472546, splitWal=true, meta=false, isMeta: false 2024-12-04T09:44:49,038 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting 2024-12-04T09:44:49,039 INFO [PEWorker-1 {}] master.SplitWALManager(105): 84486a41f81c,38215,1733305472546 WAL count=1, meta=false 2024-12-04T09:44:49,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 84486a41f81c%2C38215%2C1733305472546.1733305474029}] 2024-12-04T09:44:49,047 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=84486a41f81c,35147,1733305472454 2024-12-04T09:44:49,050 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 84486a41f81c%2C38215%2C1733305472546.1733305474029, worker=84486a41f81c,35147,1733305472454}] 2024-12-04T09:44:49,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:44:49,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38215-0x101a10432830002, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:44:49,118 INFO [RS:1;84486a41f81c:38215 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T09:44:49,118 INFO [RS:1;84486a41f81c:38215 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,38215,1733305472546; zookeeper connection closed. 2024-12-04T09:44:49,119 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@327ce93b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@327ce93b 2024-12-04T09:44:49,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35147 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-04T09:44:49,226 INFO [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029, size=1.6 K (1677bytes) 2024-12-04T09:44:49,226 INFO [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029 2024-12-04T09:44:49,226 INFO [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029 after 0ms 2024-12-04T09:44:49,229 DEBUG [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:49,229 INFO [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029 took 4ms 2024-12-04T09:44:49,235 DEBUG [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 56ae12cc64f9a720b1c872193297ac28: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-04T09:44:49,235 DEBUG [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029 so closing down 2024-12-04T09:44:49,235 DEBUG [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:49,235 INFO [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:49,236 INFO [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-04T09:44:49,236 DEBUG [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029, journal: Splitting hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029, size=1.6 K (1677bytes) at 1733305489226Finishing writing output for hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029 so closing down at 1733305489235 (+9 ms)3 split writer threads finished at 1733305489235Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733305489236 (+1 ms) 2024-12-04T09:44:49,236 DEBUG [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029 2024-12-04T09:44:49,237 DEBUG [RS_LOG_REPLAY_OPS-regionserver/84486a41f81c:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-04T09:44:49,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38097 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-04T09:44:49,242 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting/84486a41f81c%2C38215%2C1733305472546.1733305474029 to hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs 2024-12-04T09:44:49,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T09:44:49,245 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 84486a41f81c%2C38215%2C1733305472546.1733305474029, worker=84486a41f81c,35147,1733305472454 in 193 msec 2024-12-04T09:44:49,247 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=84486a41f81c,35147,1733305472454 2024-12-04T09:44:49,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T09:44:49,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 84486a41f81c%2C38215%2C1733305472546.1733305474029, worker=84486a41f81c,35147,1733305472454 in 206 msec 2024-12-04T09:44:49,252 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting dir is empty, no logs to split. 2024-12-04T09:44:49,253 INFO [PEWorker-1 {}] master.SplitWALManager(105): 84486a41f81c,38215,1733305472546 WAL count=0, meta=false 2024-12-04T09:44:49,253 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 84486a41f81c,38215,1733305472546 WAL splitting is done? wals=0, meta=false 2024-12-04T09:44:49,255 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 84486a41f81c,38215,1733305472546 failed, ignore...File hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/WALs/84486a41f81c,38215,1733305472546-splitting does not exist. 2024-12-04T09:44:49,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN}] 2024-12-04T09:44:49,258 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN 2024-12-04T09:44:49,259 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T09:44:49,338 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18] 2024-12-04T09:44:49,339 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 84486a41f81c:38215 this server is in the failed servers list 2024-12-04T09:44:49,340 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=84486a41f81c:38215 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 84486a41f81c:38215 2024-12-04T09:44:49,340 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 84486a41f81c:38215 2024-12-04T09:44:49,340 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,38215,1733305472546, seqNum=18 from cache 2024-12-04T09:44:49,410 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(204): Hosts are {84486a41f81c=0} racks are {/default-rack=0} 2024-12-04T09:44:49,410 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T09:44:49,410 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T09:44:49,410 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T09:44:49,410 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T09:44:49,411 INFO [84486a41f81c:38097 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T09:44:49,411 INFO [84486a41f81c:38097 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T09:44:49,411 DEBUG [84486a41f81c:38097 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T09:44:49,411 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPENING, regionLocation=84486a41f81c,35147,1733305472454 2024-12-04T09:44:49,415 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN because future has completed 2024-12-04T09:44:49,416 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454}] 2024-12-04T09:44:49,591 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:49,592 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 56ae12cc64f9a720b1c872193297ac28, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:49,592 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,592 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:49,592 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,593 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,594 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,596 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf1 2024-12-04T09:44:49,596 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,602 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf1/8b776c96d0d0458a9afea98c13b9a9ac 2024-12-04T09:44:49,603 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,603 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,604 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56ae12cc64f9a720b1c872193297ac28 columnFamilyName cf2 2024-12-04T09:44:49,604 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,610 DEBUG [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/cf2/52c1af6087ff4d93a5aa17f28812c17d 2024-12-04T09:44:49,611 INFO [StoreOpener-56ae12cc64f9a720b1c872193297ac28-1 {}] regionserver.HStore(327): Store=56ae12cc64f9a720b1c872193297ac28/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,611 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,612 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,613 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,614 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,614 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,615 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T09:44:49,617 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,618 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 56ae12cc64f9a720b1c872193297ac28; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73855443, jitterRate=0.10053186118602753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T09:44:49,618 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:44:49,619 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 56ae12cc64f9a720b1c872193297ac28: Running coprocessor pre-open hook at 1733305489593Writing region info on filesystem at 1733305489593Initializing all the Stores at 1733305489594 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489594Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489594Cleaning up temporary data from old regions at 1733305489614 (+20 ms)Running coprocessor post-open hooks at 1733305489618 (+4 ms)Region opened successfully at 1733305489619 (+1 ms) 2024-12-04T09:44:49,620 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., pid=17, masterSystemTime=1733305489587 2024-12-04T09:44:49,622 DEBUG [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:49,622 INFO [RS_OPEN_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:44:49,623 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=56ae12cc64f9a720b1c872193297ac28, regionState=OPEN, openSeqNum=18, regionLocation=84486a41f81c,35147,1733305472454 2024-12-04T09:44:49,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454 because future has completed 2024-12-04T09:44:49,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T09:44:49,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 56ae12cc64f9a720b1c872193297ac28, server=84486a41f81c,35147,1733305472454 in 210 msec 2024-12-04T09:44:49,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-04T09:44:49,631 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=56ae12cc64f9a720b1c872193297ac28, ASSIGN in 372 msec 2024-12-04T09:44:49,631 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 84486a41f81c,38215,1733305472546 after splitting done 2024-12-04T09:44:49,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 84486a41f81c,38215,1733305472546, splitWal=true, meta=false in 604 msec 2024-12-04T09:44:49,859 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28., hostname=84486a41f81c,35147,1733305472454, seqNum=18] 2024-12-04T09:44:49,875 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=403 (was 407), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 389), ProcessCount=11 (was 11), AvailableMemoryMB=10594 (was 10661) 2024-12-04T09:44:49,877 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-04T09:44:49,887 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=403, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=10593 2024-12-04T09:44:49,888 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-04T09:44:49,901 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:49,903 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:49,904 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:49,906 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63557148, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-63557148, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:49,923 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63557148/hregion-63557148.1733305489907, exclude list is [], retry=0 2024-12-04T09:44:49,926 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:49,926 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:49,927 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:49,929 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63557148/hregion-63557148.1733305489907 2024-12-04T09:44:49,930 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:44:49,930 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a4c6f876fd33f8a0225e01633678bf24, NAME => 'testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:49,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741893_1071 (size=67) 2024-12-04T09:44:49,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741893_1071 (size=67) 2024-12-04T09:44:49,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741893_1071 (size=67) 2024-12-04T09:44:49,942 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:49,943 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,944 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName a 2024-12-04T09:44:49,944 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,945 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,945 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,946 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName b 2024-12-04T09:44:49,947 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,947 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,947 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,948 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName c 2024-12-04T09:44:49,948 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,949 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,949 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,950 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,951 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,951 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,951 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:49,952 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,954 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:49,954 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4c6f876fd33f8a0225e01633678bf24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64388038, jitterRate=-0.04054346680641174}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:49,955 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4c6f876fd33f8a0225e01633678bf24: Writing region info on filesystem at 1733305489942Initializing all the Stores at 1733305489943 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489943Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489943Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489943Cleaning up temporary data from old regions at 1733305489951 (+8 ms)Region opened successfully at 1733305489955 (+4 ms) 2024-12-04T09:44:49,955 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4c6f876fd33f8a0225e01633678bf24, disabling compactions & flushes 2024-12-04T09:44:49,955 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:49,955 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:49,956 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. after waiting 0 ms 2024-12-04T09:44:49,956 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:49,956 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:49,956 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4c6f876fd33f8a0225e01633678bf24: Waiting for close lock at 1733305489955Disabling compacts and flushes for region at 1733305489955Disabling writes for close at 1733305489956 (+1 ms)Writing region close event to WAL at 1733305489956Closed at 1733305489956 2024-12-04T09:44:49,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741892_1070 (size=93) 2024-12-04T09:44:49,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741892_1070 (size=93) 2024-12-04T09:44:49,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741892_1070 (size=93) 2024-12-04T09:44:49,961 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:49,961 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63557148:(num 1733305489907) 2024-12-04T09:44:49,961 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:49,963 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:49,975 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963, exclude list is [], retry=0 2024-12-04T09:44:49,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:49,977 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:49,978 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:49,979 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963 2024-12-04T09:44:49,979 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:49,980 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4c6f876fd33f8a0225e01633678bf24, NAME => 'testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:49,980 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:49,980 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,980 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,981 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,982 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName a 2024-12-04T09:44:49,982 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,982 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,982 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,983 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName b 2024-12-04T09:44:49,983 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,984 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,984 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,985 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName c 2024-12-04T09:44:49,985 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:49,985 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:49,985 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,986 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,987 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,988 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,988 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,988 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:49,989 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:49,990 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4c6f876fd33f8a0225e01633678bf24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72208912, jitterRate=0.07599663734436035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:49,991 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4c6f876fd33f8a0225e01633678bf24: Writing region info on filesystem at 1733305489980Initializing all the Stores at 1733305489981 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489981Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489981Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305489981Cleaning up temporary data from old regions at 1733305489988 (+7 ms)Region opened successfully at 1733305489991 (+3 ms) 2024-12-04T09:44:50,016 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4c6f876fd33f8a0225e01633678bf24 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T09:44:50,037 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/a/a05fa838492442e391f3718d2dc39f39 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733305489991/Put/seqid=0 2024-12-04T09:44:50,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741895_1073 (size=5958) 2024-12-04T09:44:50,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741895_1073 (size=5958) 2024-12-04T09:44:50,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741895_1073 (size=5958) 2024-12-04T09:44:50,045 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/a/a05fa838492442e391f3718d2dc39f39 2024-12-04T09:44:50,070 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/b/d70683f22a6e4c1489e5743dae8951c5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733305490000/Put/seqid=0 2024-12-04T09:44:50,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741896_1074 (size=5958) 2024-12-04T09:44:50,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741896_1074 (size=5958) 2024-12-04T09:44:50,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741896_1074 (size=5958) 2024-12-04T09:44:50,078 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/b/d70683f22a6e4c1489e5743dae8951c5 2024-12-04T09:44:50,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/c/a10a67878a11449caf66c46800d16b95 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733305490007/Put/seqid=0 2024-12-04T09:44:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741897_1075 (size=5958) 2024-12-04T09:44:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741897_1075 (size=5958) 2024-12-04T09:44:50,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741897_1075 (size=5958) 2024-12-04T09:44:50,113 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/c/a10a67878a11449caf66c46800d16b95 2024-12-04T09:44:50,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/a/a05fa838492442e391f3718d2dc39f39 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/a/a05fa838492442e391f3718d2dc39f39 2024-12-04T09:44:50,129 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/a/a05fa838492442e391f3718d2dc39f39, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T09:44:50,131 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/b/d70683f22a6e4c1489e5743dae8951c5 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/b/d70683f22a6e4c1489e5743dae8951c5 2024-12-04T09:44:50,138 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/b/d70683f22a6e4c1489e5743dae8951c5, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T09:44:50,140 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/c/a10a67878a11449caf66c46800d16b95 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/c/a10a67878a11449caf66c46800d16b95 2024-12-04T09:44:50,147 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/c/a10a67878a11449caf66c46800d16b95, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T09:44:50,149 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for a4c6f876fd33f8a0225e01633678bf24 in 133ms, sequenceid=33, compaction requested=false 2024-12-04T09:44:50,149 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a4c6f876fd33f8a0225e01633678bf24: 2024-12-04T09:44:50,149 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a4c6f876fd33f8a0225e01633678bf24, disabling compactions & flushes 2024-12-04T09:44:50,149 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:50,149 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:50,149 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. after waiting 0 ms 2024-12-04T09:44:50,149 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:50,151 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. 2024-12-04T09:44:50,151 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a4c6f876fd33f8a0225e01633678bf24: Waiting for close lock at 1733305490149Disabling compacts and flushes for region at 1733305490149Disabling writes for close at 1733305490149Writing region close event to WAL at 1733305490151 (+2 ms)Closed at 1733305490151 2024-12-04T09:44:50,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741894_1072 (size=2873) 2024-12-04T09:44:50,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741894_1072 (size=2873) 2024-12-04T09:44:50,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741894_1072 (size=2873) 2024-12-04T09:44:50,159 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/b/d70683f22a6e4c1489e5743dae8951c5 to hdfs://localhost:33391/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/b/d70683f22a6e4c1489e5743dae8951c5 2024-12-04T09:44:50,182 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963, size=2.8 K (2873bytes) 2024-12-04T09:44:50,182 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963 2024-12-04T09:44:50,183 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963 after 1ms 2024-12-04T09:44:50,185 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:50,186 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963 took 3ms 2024-12-04T09:44:50,188 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963 so closing down 2024-12-04T09:44:50,189 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:50,189 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733305489963.temp 2024-12-04T09:44:50,191 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000003-wal.1733305489963.temp 2024-12-04T09:44:50,191 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:50,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741898_1076 (size=2312) 2024-12-04T09:44:50,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741898_1076 (size=2312) 2024-12-04T09:44:50,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741898_1076 (size=2312) 2024-12-04T09:44:50,201 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000003-wal.1733305489963.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:50,203 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000003-wal.1733305489963.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000032 2024-12-04T09:44:50,203 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-04T09:44:50,203 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963, journal: Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963, size=2.8 K (2873bytes) at 1733305490182Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963 so closing down at 1733305490188 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000003-wal.1733305489963.temp at 1733305490191 (+3 ms)3 split writer threads finished at 1733305490191Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000003-wal.1733305489963.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733305490201 (+10 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000003-wal.1733305489963.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000032 at 1733305490203 (+2 ms)Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733305490203 2024-12-04T09:44:50,205 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305489963 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305489963 2024-12-04T09:44:50,206 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000032 2024-12-04T09:44:50,206 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:50,208 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:50,220 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305490208, exclude list is [], retry=0 2024-12-04T09:44:50,223 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:50,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:50,224 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:50,227 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305490208 2024-12-04T09:44:50,228 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:44:50,228 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a4c6f876fd33f8a0225e01633678bf24, NAME => 'testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:50,228 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:50,228 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,228 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,229 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,230 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName a 2024-12-04T09:44:50,230 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,236 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/a/a05fa838492442e391f3718d2dc39f39 2024-12-04T09:44:50,236 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,237 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,238 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName b 2024-12-04T09:44:50,238 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,238 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,239 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,239 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a4c6f876fd33f8a0225e01633678bf24 columnFamilyName c 2024-12-04T09:44:50,239 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,245 DEBUG [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/c/a10a67878a11449caf66c46800d16b95 2024-12-04T09:44:50,245 INFO [StoreOpener-a4c6f876fd33f8a0225e01633678bf24-1 {}] regionserver.HStore(327): Store=a4c6f876fd33f8a0225e01633678bf24/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,245 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,246 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,248 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,249 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000032 2024-12-04T09:44:50,251 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:50,253 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000032 2024-12-04T09:44:50,253 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a4c6f876fd33f8a0225e01633678bf24 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T09:44:50,267 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/b/c3e9936cf9794e4fa451c54009df5efb is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733305490000/Put/seqid=0 2024-12-04T09:44:50,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741900_1078 (size=5958) 2024-12-04T09:44:50,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741900_1078 (size=5958) 2024-12-04T09:44:50,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741900_1078 (size=5958) 2024-12-04T09:44:50,275 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/b/c3e9936cf9794e4fa451c54009df5efb 2024-12-04T09:44:50,281 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/.tmp/b/c3e9936cf9794e4fa451c54009df5efb as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/b/c3e9936cf9794e4fa451c54009df5efb 2024-12-04T09:44:50,286 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/b/c3e9936cf9794e4fa451c54009df5efb, entries=10, sequenceid=32, filesize=5.8 K 2024-12-04T09:44:50,286 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a4c6f876fd33f8a0225e01633678bf24 in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-12-04T09:44:50,287 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/0000000000000000032 2024-12-04T09:44:50,288 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,289 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,289 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:50,291 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a4c6f876fd33f8a0225e01633678bf24 2024-12-04T09:44:50,293 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/a4c6f876fd33f8a0225e01633678bf24/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-04T09:44:50,294 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a4c6f876fd33f8a0225e01633678bf24; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75401195, jitterRate=0.12356536090373993}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:50,294 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a4c6f876fd33f8a0225e01633678bf24: Writing region info on filesystem at 1733305490228Initializing all the Stores at 1733305490229 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490229Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490229Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490229Obtaining lock to block concurrent updates at 1733305490253 (+24 ms)Preparing flush snapshotting stores in a4c6f876fd33f8a0225e01633678bf24 at 1733305490253Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733305490253Flushing stores of testReplayEditsWrittenViaHRegion,,1733305489902.a4c6f876fd33f8a0225e01633678bf24. at 1733305490253Flushing a4c6f876fd33f8a0225e01633678bf24/b: creating writer at 1733305490253Flushing a4c6f876fd33f8a0225e01633678bf24/b: appending metadata at 1733305490267 (+14 ms)Flushing a4c6f876fd33f8a0225e01633678bf24/b: closing flushed file at 1733305490267Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2779f0e7: reopening flushed file at 1733305490280 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a4c6f876fd33f8a0225e01633678bf24 in 33ms, sequenceid=32, compaction requested=false; wal=null at 1733305490287 (+7 ms)Cleaning up temporary data from old regions at 1733305490289 (+2 ms)Region opened successfully at 1733305490294 (+5 ms) 2024-12-04T09:44:50,315 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=412 (was 403) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:44374 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:51908 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:35008 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:60482 [Waiting for operation #29] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:45364 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:59576 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=10590 (was 10593) 2024-12-04T09:44:50,315 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-04T09:44:50,328 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=412, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=10584 2024-12-04T09:44:50,328 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-04T09:44:50,342 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:50,344 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:50,344 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:50,347 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63539894, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-63539894, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:50,360 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63539894/hregion-63539894.1733305490348, exclude list is [], retry=0 2024-12-04T09:44:50,363 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:50,363 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:50,364 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:50,366 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63539894/hregion-63539894.1733305490348 2024-12-04T09:44:50,366 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:50,367 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5a93c3e000ffebf405fc9a9932608cce, NAME => 'testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:50,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741902_1080 (size=68) 2024-12-04T09:44:50,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741902_1080 (size=68) 2024-12-04T09:44:50,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741902_1080 (size=68) 2024-12-04T09:44:50,386 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:50,388 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,389 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName a 2024-12-04T09:44:50,390 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,390 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,390 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,391 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName b 2024-12-04T09:44:50,392 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,392 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,392 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,394 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName c 2024-12-04T09:44:50,394 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,394 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,394 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,395 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,396 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,397 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,397 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,397 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:50,399 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,401 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:50,401 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a93c3e000ffebf405fc9a9932608cce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67480797, jitterRate=0.00554223358631134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:50,402 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a93c3e000ffebf405fc9a9932608cce: Writing region info on filesystem at 1733305490387Initializing all the Stores at 1733305490387Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490387Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490388 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490388Cleaning up temporary data from old regions at 1733305490397 (+9 ms)Region opened successfully at 1733305490402 (+5 ms) 2024-12-04T09:44:50,402 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5a93c3e000ffebf405fc9a9932608cce, disabling compactions & flushes 2024-12-04T09:44:50,402 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:50,402 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:50,402 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. after waiting 0 ms 2024-12-04T09:44:50,402 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:50,403 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:50,403 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5a93c3e000ffebf405fc9a9932608cce: Waiting for close lock at 1733305490402Disabling compacts and flushes for region at 1733305490402Disabling writes for close at 1733305490402Writing region close event to WAL at 1733305490403 (+1 ms)Closed at 1733305490403 2024-12-04T09:44:50,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741901_1079 (size=93) 2024-12-04T09:44:50,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741901_1079 (size=93) 2024-12-04T09:44:50,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741901_1079 (size=93) 2024-12-04T09:44:50,408 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:50,408 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63539894:(num 1733305490348) 2024-12-04T09:44:50,409 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:50,410 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:50,422 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411, exclude list is [], retry=0 2024-12-04T09:44:50,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:50,425 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:50,426 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:50,428 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411 2024-12-04T09:44:50,428 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:50,501 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a93c3e000ffebf405fc9a9932608cce, NAME => 'testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:50,503 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,503 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:50,504 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,504 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,506 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,507 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName a 2024-12-04T09:44:50,507 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,507 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,508 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,508 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName b 2024-12-04T09:44:50,509 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,509 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,509 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,510 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName c 2024-12-04T09:44:50,510 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:50,511 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:50,511 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,512 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,514 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,514 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,515 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:50,516 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,517 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a93c3e000ffebf405fc9a9932608cce; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61899397, jitterRate=-0.07762710750102997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:50,517 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:50,518 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a93c3e000ffebf405fc9a9932608cce: Running coprocessor pre-open hook at 1733305490504Writing region info on filesystem at 1733305490504Initializing all the Stores at 1733305490505 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490505Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490505Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305490505Cleaning up temporary data from old regions at 1733305490514 (+9 ms)Running coprocessor post-open hooks at 1733305490517 (+3 ms)Region opened successfully at 1733305490518 (+1 ms) 2024-12-04T09:44:50,531 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5a93c3e000ffebf405fc9a9932608cce 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-04T09:44:50,532 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:51,533 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:52,371 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T09:44:52,412 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-04T09:44:52,413 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:52,413 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T09:44:52,413 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:52,414 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-04T09:44:52,414 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:52,415 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-04T09:44:52,415 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-04T09:44:52,535 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:53,536 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:54,537 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:55,539 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:56,540 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:57,541 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:58,541 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:59,542 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 5a93c3e000ffebf405fc9a9932608cce/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:44:59,544 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5a93c3e000ffebf405fc9a9932608cce: 2024-12-04T09:44:59,544 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:59,560 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5a93c3e000ffebf405fc9a9932608cce: 2024-12-04T09:44:59,561 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-04T09:44:59,561 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5a93c3e000ffebf405fc9a9932608cce, disabling compactions & flushes 2024-12-04T09:44:59,561 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:59,561 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:59,561 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. after waiting 0 ms 2024-12-04T09:44:59,561 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:59,561 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:59,562 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. 2024-12-04T09:44:59,562 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5a93c3e000ffebf405fc9a9932608cce: Waiting for close lock at 1733305499561Running coprocessor pre-close hooks at 1733305499561Disabling compacts and flushes for region at 1733305499561Disabling writes for close at 1733305499561Writing region close event to WAL at 1733305499561Running coprocessor post-close hooks at 1733305499561Closed at 1733305499561 2024-12-04T09:44:59,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741903_1081 (size=2398) 2024-12-04T09:44:59,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741903_1081 (size=2398) 2024-12-04T09:44:59,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741903_1081 (size=2398) 2024-12-04T09:44:59,580 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411, size=2.3 K (2398bytes) 2024-12-04T09:44:59,580 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411 2024-12-04T09:44:59,580 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411 after 0ms 2024-12-04T09:44:59,583 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:59,583 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411 took 4ms 2024-12-04T09:44:59,586 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411 so closing down 2024-12-04T09:44:59,586 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:44:59,587 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733305490411.temp 2024-12-04T09:44:59,589 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000004-wal.1733305490411.temp 2024-12-04T09:44:59,589 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:44:59,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741904_1082 (size=1672) 2024-12-04T09:44:59,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741904_1082 (size=1672) 2024-12-04T09:44:59,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741904_1082 (size=1672) 2024-12-04T09:44:59,599 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000004-wal.1733305490411.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-04T09:44:59,601 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000004-wal.1733305490411.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000026 2024-12-04T09:44:59,601 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-04T09:44:59,601 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411, journal: Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411, size=2.3 K (2398bytes) at 1733305499580Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411 so closing down at 1733305499586 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000004-wal.1733305490411.temp at 1733305499589 (+3 ms)3 split writer threads finished at 1733305499589Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000004-wal.1733305490411.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733305499600 (+11 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000004-wal.1733305490411.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000026 at 1733305499601 (+1 ms)Processed 23 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733305499601 2024-12-04T09:44:59,603 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305490411 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305490411 2024-12-04T09:44:59,603 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000026 2024-12-04T09:44:59,604 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:59,606 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:59,621 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305499606, exclude list is [], retry=0 2024-12-04T09:44:59,624 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:59,624 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:59,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:59,628 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305499606 2024-12-04T09:44:59,628 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:59,629 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5a93c3e000ffebf405fc9a9932608cce, NAME => 'testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:59,629 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,629 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:59,629 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,629 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,635 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,636 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName a 2024-12-04T09:44:59,636 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:59,637 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:59,637 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,638 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName b 2024-12-04T09:44:59,638 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:59,639 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:59,639 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,640 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5a93c3e000ffebf405fc9a9932608cce columnFamilyName c 2024-12-04T09:44:59,640 DEBUG [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:59,641 INFO [StoreOpener-5a93c3e000ffebf405fc9a9932608cce-1 {}] regionserver.HStore(327): Store=5a93c3e000ffebf405fc9a9932608cce/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:59,641 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,641 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,643 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000026 2024-12-04T09:44:59,645 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:59,647 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000026 2024-12-04T09:44:59,648 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5a93c3e000ffebf405fc9a9932608cce 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-04T09:44:59,664 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/a/8c15bebc486a41d786586e53f11f5556 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733305499549/Put/seqid=0 2024-12-04T09:44:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741906_1084 (size=5523) 2024-12-04T09:44:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741906_1084 (size=5523) 2024-12-04T09:44:59,675 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/a/8c15bebc486a41d786586e53f11f5556 2024-12-04T09:44:59,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741906_1084 (size=5523) 2024-12-04T09:44:59,705 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/b/5139bfe74f5041a580d326372bc51441 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733305499544/Put/seqid=0 2024-12-04T09:44:59,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741907_1085 (size=5524) 2024-12-04T09:44:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741907_1085 (size=5524) 2024-12-04T09:44:59,713 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/b/5139bfe74f5041a580d326372bc51441 2024-12-04T09:44:59,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741907_1085 (size=5524) 2024-12-04T09:44:59,741 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/c/212e92ad07b04798a27a1e1b5836b0a3 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733305499547/Put/seqid=0 2024-12-04T09:44:59,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741908_1086 (size=5457) 2024-12-04T09:44:59,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741908_1086 (size=5457) 2024-12-04T09:44:59,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741908_1086 (size=5457) 2024-12-04T09:44:59,749 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/c/212e92ad07b04798a27a1e1b5836b0a3 2024-12-04T09:44:59,755 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/a/8c15bebc486a41d786586e53f11f5556 as hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/a/8c15bebc486a41d786586e53f11f5556 2024-12-04T09:44:59,761 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/a/8c15bebc486a41d786586e53f11f5556, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T09:44:59,763 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/b/5139bfe74f5041a580d326372bc51441 as hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/b/5139bfe74f5041a580d326372bc51441 2024-12-04T09:44:59,769 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/b/5139bfe74f5041a580d326372bc51441, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T09:44:59,770 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/.tmp/c/212e92ad07b04798a27a1e1b5836b0a3 as hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/c/212e92ad07b04798a27a1e1b5836b0a3 2024-12-04T09:44:59,776 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/c/212e92ad07b04798a27a1e1b5836b0a3, entries=6, sequenceid=26, filesize=5.3 K 2024-12-04T09:44:59,776 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 5a93c3e000ffebf405fc9a9932608cce in 129ms, sequenceid=26, compaction requested=false; wal=null 2024-12-04T09:44:59,777 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/0000000000000000026 2024-12-04T09:44:59,778 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,778 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,779 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:44:59,781 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,783 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsAfterAbortingFlush/5a93c3e000ffebf405fc9a9932608cce/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-04T09:44:59,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5a93c3e000ffebf405fc9a9932608cce; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74250840, jitterRate=0.10642373561859131}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:44:59,785 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5a93c3e000ffebf405fc9a9932608cce 2024-12-04T09:44:59,786 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5a93c3e000ffebf405fc9a9932608cce: Running coprocessor pre-open hook at 1733305499630Writing region info on filesystem at 1733305499630Initializing all the Stores at 1733305499631 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305499631Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305499634 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305499634Obtaining lock to block concurrent updates at 1733305499648 (+14 ms)Preparing flush snapshotting stores in 5a93c3e000ffebf405fc9a9932608cce at 1733305499648Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733305499648Flushing stores of testReplayEditsAfterAbortingFlush,,1733305490342.5a93c3e000ffebf405fc9a9932608cce. at 1733305499648Flushing 5a93c3e000ffebf405fc9a9932608cce/a: creating writer at 1733305499648Flushing 5a93c3e000ffebf405fc9a9932608cce/a: appending metadata at 1733305499664 (+16 ms)Flushing 5a93c3e000ffebf405fc9a9932608cce/a: closing flushed file at 1733305499664Flushing 5a93c3e000ffebf405fc9a9932608cce/b: creating writer at 1733305499681 (+17 ms)Flushing 5a93c3e000ffebf405fc9a9932608cce/b: appending metadata at 1733305499704 (+23 ms)Flushing 5a93c3e000ffebf405fc9a9932608cce/b: closing flushed file at 1733305499705 (+1 ms)Flushing 5a93c3e000ffebf405fc9a9932608cce/c: creating writer at 1733305499720 (+15 ms)Flushing 5a93c3e000ffebf405fc9a9932608cce/c: appending metadata at 1733305499740 (+20 ms)Flushing 5a93c3e000ffebf405fc9a9932608cce/c: closing flushed file at 1733305499740Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d7caed8: reopening flushed file at 1733305499754 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c9e4bde: reopening flushed file at 1733305499762 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@544ec9ea: reopening flushed file at 1733305499769 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 5a93c3e000ffebf405fc9a9932608cce in 129ms, sequenceid=26, compaction requested=false; wal=null at 1733305499776 (+7 ms)Cleaning up temporary data from old regions at 1733305499778 (+2 ms)Running coprocessor post-open hooks at 1733305499785 (+7 ms)Region opened successfully at 1733305499786 (+1 ms) 2024-12-04T09:44:59,806 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=413 (was 412) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:38868 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:38856 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:42314 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:53156 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:42292 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:53180 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=10425 (was 10584) 2024-12-04T09:44:59,806 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-04T09:44:59,818 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=10424 2024-12-04T09:44:59,818 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-04T09:44:59,833 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:59,835 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:44:59,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:44:59,839 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-84184788, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-84184788, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:59,852 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-84184788/hregion-84184788.1733305499839, exclude list is [], retry=0 2024-12-04T09:44:59,855 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:59,855 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:59,860 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:59,862 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-84184788/hregion-84184788.1733305499839 2024-12-04T09:44:59,863 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:59,863 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6490b8684536253e684f5251db6dca52, NAME => 'testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:44:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741910_1088 (size=61) 2024-12-04T09:44:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741910_1088 (size=61) 2024-12-04T09:44:59,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741910_1088 (size=61) 2024-12-04T09:44:59,892 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:59,900 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,908 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6490b8684536253e684f5251db6dca52 columnFamilyName a 2024-12-04T09:44:59,908 DEBUG [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:59,909 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(327): Store=6490b8684536253e684f5251db6dca52/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:59,910 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,912 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,912 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,913 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,916 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:44:59,917 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6490b8684536253e684f5251db6dca52; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67023090, jitterRate=-0.0012781322002410889}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:44:59,918 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6490b8684536253e684f5251db6dca52: Writing region info on filesystem at 1733305499892Initializing all the Stores at 1733305499893 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305499893Cleaning up temporary data from old regions at 1733305499912 (+19 ms)Region opened successfully at 1733305499918 (+6 ms) 2024-12-04T09:44:59,918 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6490b8684536253e684f5251db6dca52, disabling compactions & flushes 2024-12-04T09:44:59,918 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,919 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,919 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. after waiting 0 ms 2024-12-04T09:44:59,919 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,927 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,927 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6490b8684536253e684f5251db6dca52: Waiting for close lock at 1733305499918Disabling compacts and flushes for region at 1733305499918Disabling writes for close at 1733305499919 (+1 ms)Writing region close event to WAL at 1733305499927 (+8 ms)Closed at 1733305499927 2024-12-04T09:44:59,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741909_1087 (size=93) 2024-12-04T09:44:59,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741909_1087 (size=93) 2024-12-04T09:44:59,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741909_1087 (size=93) 2024-12-04T09:44:59,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:44:59,934 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-84184788:(num 1733305499839) 2024-12-04T09:44:59,934 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:44:59,936 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:44:59,953 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936, exclude list is [], retry=0 2024-12-04T09:44:59,956 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:44:59,957 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:44:59,957 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:44:59,958 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936 2024-12-04T09:44:59,958 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:44:59,959 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6490b8684536253e684f5251db6dca52, NAME => 'testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:44:59,959 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:44:59,959 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,959 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,963 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,964 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6490b8684536253e684f5251db6dca52 columnFamilyName a 2024-12-04T09:44:59,964 DEBUG [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:44:59,965 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(327): Store=6490b8684536253e684f5251db6dca52/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:44:59,965 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,965 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,967 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,967 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,969 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6490b8684536253e684f5251db6dca52 2024-12-04T09:44:59,970 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6490b8684536253e684f5251db6dca52; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72704153, jitterRate=0.08337630331516266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:44:59,970 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6490b8684536253e684f5251db6dca52: Writing region info on filesystem at 1733305499959Initializing all the Stores at 1733305499960 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305499960Cleaning up temporary data from old regions at 1733305499967 (+7 ms)Region opened successfully at 1733305499970 (+3 ms) 2024-12-04T09:44:59,979 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6490b8684536253e684f5251db6dca52, disabling compactions & flushes 2024-12-04T09:44:59,979 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,979 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,979 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. after waiting 0 ms 2024-12-04T09:44:59,979 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,980 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,980 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. 2024-12-04T09:44:59,980 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6490b8684536253e684f5251db6dca52: Waiting for close lock at 1733305499979Disabling compacts and flushes for region at 1733305499979Disabling writes for close at 1733305499979Writing region close event to WAL at 1733305499979Closed at 1733305499980 (+1 ms) 2024-12-04T09:44:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741911_1089 (size=838) 2024-12-04T09:44:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741911_1089 (size=838) 2024-12-04T09:44:59,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741911_1089 (size=838) 2024-12-04T09:44:59,996 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936, size=838 (838bytes) 2024-12-04T09:44:59,996 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936 2024-12-04T09:44:59,996 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936 after 0ms 2024-12-04T09:44:59,998 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:44:59,998 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936 took 3ms 2024-12-04T09:44:59,999 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936 so closing down 2024-12-04T09:44:59,999 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:45:00,001 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733305499936.temp 2024-12-04T09:45:00,002 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000003-wal.1733305499936.temp 2024-12-04T09:45:00,002 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:45:00,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741912_1090 (size=838) 2024-12-04T09:45:00,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741912_1090 (size=838) 2024-12-04T09:45:00,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741912_1090 (size=838) 2024-12-04T09:45:00,008 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000003-wal.1733305499936.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-04T09:45:00,009 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000003-wal.1733305499936.temp to hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012 2024-12-04T09:45:00,009 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936, size=838, length=838, corrupted=false, cancelled=false 2024-12-04T09:45:00,010 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936, journal: Splitting hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936, size=838 (838bytes) at 1733305499996Finishing writing output for hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936 so closing down at 1733305499999 (+3 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000003-wal.1733305499936.temp at 1733305500002 (+3 ms)3 split writer threads finished at 1733305500002Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000003-wal.1733305499936.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733305500008 (+6 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000003-wal.1733305499936.temp to hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012 at 1733305500009 (+1 ms)Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936, size=838, length=838, corrupted=false, cancelled=false at 1733305500009 2024-12-04T09:45:00,011 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305499936 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305499936 2024-12-04T09:45:00,012 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012 2024-12-04T09:45:00,016 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:00,346 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:45:00,348 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:00,361 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305500348, exclude list is [], retry=0 2024-12-04T09:45:00,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:00,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:00,364 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:00,366 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305500348 2024-12-04T09:45:00,366 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:45:00,366 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6490b8684536253e684f5251db6dca52, NAME => 'testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:45:00,366 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:00,366 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,366 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,369 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,370 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6490b8684536253e684f5251db6dca52 columnFamilyName a 2024-12-04T09:45:00,370 DEBUG [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,371 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(327): Store=6490b8684536253e684f5251db6dca52/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,371 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,372 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,374 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012 2024-12-04T09:45:00,376 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:00,377 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012 2024-12-04T09:45:00,377 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6490b8684536253e684f5251db6dca52 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-04T09:45:00,394 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/.tmp/a/3f9d9f237b3d40ed815f1d9850d721fa is 79, key is testDatalossWhenInputError/a:x0/1733305499970/Put/seqid=0 2024-12-04T09:45:00,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741914_1092 (size=5808) 2024-12-04T09:45:00,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741914_1092 (size=5808) 2024-12-04T09:45:00,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741914_1092 (size=5808) 2024-12-04T09:45:00,401 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/.tmp/a/3f9d9f237b3d40ed815f1d9850d721fa 2024-12-04T09:45:00,410 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/.tmp/a/3f9d9f237b3d40ed815f1d9850d721fa as hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/a/3f9d9f237b3d40ed815f1d9850d721fa 2024-12-04T09:45:00,421 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/a/3f9d9f237b3d40ed815f1d9850d721fa, entries=10, sequenceid=12, filesize=5.7 K 2024-12-04T09:45:00,421 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6490b8684536253e684f5251db6dca52 in 44ms, sequenceid=12, compaction requested=false; wal=null 2024-12-04T09:45:00,422 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/0000000000000000012 2024-12-04T09:45:00,423 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,423 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,426 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,428 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-04T09:45:00,429 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6490b8684536253e684f5251db6dca52; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71512122, jitterRate=0.06561365723609924}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:45:00,430 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6490b8684536253e684f5251db6dca52: Writing region info on filesystem at 1733305500366Initializing all the Stores at 1733305500369 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500369Obtaining lock to block concurrent updates at 1733305500377 (+8 ms)Preparing flush snapshotting stores in 6490b8684536253e684f5251db6dca52 at 1733305500377Finished memstore snapshotting testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733305500377Flushing stores of testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52. at 1733305500377Flushing 6490b8684536253e684f5251db6dca52/a: creating writer at 1733305500377Flushing 6490b8684536253e684f5251db6dca52/a: appending metadata at 1733305500393 (+16 ms)Flushing 6490b8684536253e684f5251db6dca52/a: closing flushed file at 1733305500393Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fb50cec: reopening flushed file at 1733305500409 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6490b8684536253e684f5251db6dca52 in 44ms, sequenceid=12, compaction requested=false; wal=null at 1733305500421 (+12 ms)Cleaning up temporary data from old regions at 1733305500423 (+2 ms)Region opened successfully at 1733305500430 (+7 ms) 2024-12-04T09:45:00,433 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6490b8684536253e684f5251db6dca52, NAME => 'testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:45:00,433 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733305499833.6490b8684536253e684f5251db6dca52.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:00,433 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,433 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,434 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,435 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6490b8684536253e684f5251db6dca52 columnFamilyName a 2024-12-04T09:45:00,435 DEBUG [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,441 DEBUG [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/a/3f9d9f237b3d40ed815f1d9850d721fa 2024-12-04T09:45:00,441 INFO [StoreOpener-6490b8684536253e684f5251db6dca52-1 {}] regionserver.HStore(327): Store=6490b8684536253e684f5251db6dca52/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,441 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,442 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,443 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,444 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,444 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,446 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6490b8684536253e684f5251db6dca52 2024-12-04T09:45:00,449 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testDatalossWhenInputError/6490b8684536253e684f5251db6dca52/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-04T09:45:00,450 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6490b8684536253e684f5251db6dca52; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62452022, jitterRate=-0.06939235329627991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T09:45:00,450 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6490b8684536253e684f5251db6dca52: Writing region info on filesystem at 1733305500433Initializing all the Stores at 1733305500434 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500434Cleaning up temporary data from old regions at 1733305500444 (+10 ms)Region opened successfully at 1733305500450 (+6 ms) 2024-12-04T09:45:00,468 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=423 (was 413) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:42378 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:38868 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:42292 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:53238 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:53180 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:38946 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-129416318-172.17.0.2-1733305468255:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 365), ProcessCount=11 (was 11), AvailableMemoryMB=10393 (was 10424) 2024-12-04T09:45:00,469 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-04T09:45:00,482 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=10392 2024-12-04T09:45:00,482 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-04T09:45:00,494 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:45:00,496 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:45:00,497 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:45:00,499 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-97484309, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-97484309, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:00,512 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-97484309/hregion-97484309.1733305500499, exclude list is [], retry=0 2024-12-04T09:45:00,516 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:00,516 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:00,517 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:00,530 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-97484309/hregion-97484309.1733305500499 2024-12-04T09:45:00,531 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:45:00,531 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d0c740b9030c1aa9c3838949a193179b, NAME => 'testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:45:00,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741916_1094 (size=63) 2024-12-04T09:45:00,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741916_1094 (size=63) 2024-12-04T09:45:00,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741916_1094 (size=63) 2024-12-04T09:45:00,547 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:00,548 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,550 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName a 2024-12-04T09:45:00,550 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,551 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,551 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,553 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName b 2024-12-04T09:45:00,553 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,554 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,554 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,556 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName c 2024-12-04T09:45:00,556 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,557 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,557 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,558 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,564 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,564 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,565 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:45:00,567 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,581 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:45:00,582 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0c740b9030c1aa9c3838949a193179b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74066406, jitterRate=0.10367545485496521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:45:00,582 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0c740b9030c1aa9c3838949a193179b: Writing region info on filesystem at 1733305500547Initializing all the Stores at 1733305500548 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500548Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500548Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500548Cleaning up temporary data from old regions at 1733305500564 (+16 ms)Region opened successfully at 1733305500582 (+18 ms) 2024-12-04T09:45:00,582 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0c740b9030c1aa9c3838949a193179b, disabling compactions & flushes 2024-12-04T09:45:00,582 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:00,582 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:00,582 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. after waiting 0 ms 2024-12-04T09:45:00,582 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:00,583 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:00,583 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0c740b9030c1aa9c3838949a193179b: Waiting for close lock at 1733305500582Disabling compacts and flushes for region at 1733305500582Disabling writes for close at 1733305500582Writing region close event to WAL at 1733305500583 (+1 ms)Closed at 1733305500583 2024-12-04T09:45:00,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741915_1093 (size=93) 2024-12-04T09:45:00,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741915_1093 (size=93) 2024-12-04T09:45:00,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741915_1093 (size=93) 2024-12-04T09:45:00,587 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:45:00,588 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-97484309:(num 1733305500499) 2024-12-04T09:45:00,588 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:45:00,589 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:00,601 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590, exclude list is [], retry=0 2024-12-04T09:45:00,603 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:00,604 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:00,604 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:00,606 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 2024-12-04T09:45:00,606 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:45:00,607 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0c740b9030c1aa9c3838949a193179b, NAME => 'testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:45:00,607 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:00,607 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,607 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,609 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,610 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName a 2024-12-04T09:45:00,610 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,611 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,611 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,613 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName b 2024-12-04T09:45:00,613 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,613 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,614 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,614 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName c 2024-12-04T09:45:00,615 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:00,616 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:00,616 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,617 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,618 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,620 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,620 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,621 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:45:00,623 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:00,625 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0c740b9030c1aa9c3838949a193179b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59066736, jitterRate=-0.11983704566955566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:45:00,625 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0c740b9030c1aa9c3838949a193179b: Writing region info on filesystem at 1733305500607Initializing all the Stores at 1733305500608 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500608Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500609 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305500609Cleaning up temporary data from old regions at 1733305500620 (+11 ms)Region opened successfully at 1733305500625 (+5 ms) 2024-12-04T09:45:00,632 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733305500631/Put/seqid=0 2024-12-04T09:45:00,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741918_1096 (size=4875) 2024-12-04T09:45:00,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741918_1096 (size=4875) 2024-12-04T09:45:00,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741918_1096 (size=4875) 2024-12-04T09:45:00,658 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733305500658/Put/seqid=0 2024-12-04T09:45:00,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741919_1097 (size=4875) 2024-12-04T09:45:00,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741919_1097 (size=4875) 2024-12-04T09:45:00,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741919_1097 (size=4875) 2024-12-04T09:45:00,925 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T09:45:01,068 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733305501067/Put/seqid=0 2024-12-04T09:45:01,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741920_1098 (size=4875) 2024-12-04T09:45:01,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741920_1098 (size=4875) 2024-12-04T09:45:01,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741920_1098 (size=4875) 2024-12-04T09:45:01,085 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in d0c740b9030c1aa9c3838949a193179b/a 2024-12-04T09:45:01,089 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-04T09:45:01,089 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T09:45:01,089 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in d0c740b9030c1aa9c3838949a193179b/a 2024-12-04T09:45:01,093 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-04T09:45:01,093 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T09:45:01,093 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in d0c740b9030c1aa9c3838949a193179b/a 2024-12-04T09:45:01,098 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-04T09:45:01,098 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T09:45:01,098 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0c740b9030c1aa9c3838949a193179b 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-04T09:45:01,117 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/.tmp/a/bffd9ceb4e3a482ea84300c3a7de4508 is 55, key is testCompactedBulkLoadedFiles/a:a/1733305500626/Put/seqid=0 2024-12-04T09:45:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741921_1099 (size=5107) 2024-12-04T09:45:01,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741921_1099 (size=5107) 2024-12-04T09:45:01,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741921_1099 (size=5107) 2024-12-04T09:45:01,124 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/.tmp/a/bffd9ceb4e3a482ea84300c3a7de4508 2024-12-04T09:45:01,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/.tmp/a/bffd9ceb4e3a482ea84300c3a7de4508 as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508 2024-12-04T09:45:01,135 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508, entries=1, sequenceid=4, filesize=5.0 K 2024-12-04T09:45:01,136 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for d0c740b9030c1aa9c3838949a193179b in 38ms, sequenceid=4, compaction requested=false 2024-12-04T09:45:01,136 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d0c740b9030c1aa9c3838949a193179b: 2024-12-04T09:45:01,137 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_ 2024-12-04T09:45:01,138 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_ 2024-12-04T09:45:01,139 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_ 2024-12-04T09:45:01,139 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile0 into d0c740b9030c1aa9c3838949a193179b/a as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_ - updating store file list. 2024-12-04T09:45:01,144 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1162039237864fa5849b56d66c2bfba1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T09:45:01,144 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_ into d0c740b9030c1aa9c3838949a193179b/a 2024-12-04T09:45:01,144 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile0 into d0c740b9030c1aa9c3838949a193179b/a (new location: hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_) 2024-12-04T09:45:01,145 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile1 into d0c740b9030c1aa9c3838949a193179b/a as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_ - updating store file list. 2024-12-04T09:45:01,149 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4f656f2365a149ec987f8a2118005118_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T09:45:01,149 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_ into d0c740b9030c1aa9c3838949a193179b/a 2024-12-04T09:45:01,149 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile1 into d0c740b9030c1aa9c3838949a193179b/a (new location: hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_) 2024-12-04T09:45:01,150 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile2 into d0c740b9030c1aa9c3838949a193179b/a as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_ - updating store file list. 2024-12-04T09:45:01,154 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T09:45:01,155 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_ into d0c740b9030c1aa9c3838949a193179b/a 2024-12-04T09:45:01,155 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33391/hbase/testCompactedBulkLoadedFiles/hfile2 into d0c740b9030c1aa9c3838949a193179b/a (new location: hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_) 2024-12-04T09:45:01,162 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T09:45:01,162 DEBUG [Time-limited test {}] regionserver.HStore(1541): d0c740b9030c1aa9c3838949a193179b/a is initiating major compaction (all files) 2024-12-04T09:45:01,163 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of d0c740b9030c1aa9c3838949a193179b/a in testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:01,163 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_] into tmpdir=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/.tmp, totalSize=19.3 K 2024-12-04T09:45:01,163 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bffd9ceb4e3a482ea84300c3a7de4508, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733305500626 2024-12-04T09:45:01,164 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1162039237864fa5849b56d66c2bfba1_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T09:45:01,164 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4f656f2365a149ec987f8a2118005118_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T09:45:01,165 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T09:45:01,178 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/.tmp/a/d723107983ed44caa9dc61ad95d12a61 is 55, key is testCompactedBulkLoadedFiles/a:a/1733305500626/Put/seqid=0 2024-12-04T09:45:01,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741922_1100 (size=6154) 2024-12-04T09:45:01,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741922_1100 (size=6154) 2024-12-04T09:45:01,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741922_1100 (size=6154) 2024-12-04T09:45:01,190 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/.tmp/a/d723107983ed44caa9dc61ad95d12a61 as hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/d723107983ed44caa9dc61ad95d12a61 2024-12-04T09:45:01,196 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in d0c740b9030c1aa9c3838949a193179b/a of d0c740b9030c1aa9c3838949a193179b into d723107983ed44caa9dc61ad95d12a61(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T09:45:01,196 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for d0c740b9030c1aa9c3838949a193179b: 2024-12-04T09:45:01,196 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T09:45:01,196 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T09:45:01,224 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590, size=0 (0bytes) 2024-12-04T09:45:01,224 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 might be still open, length is 0 2024-12-04T09:45:01,224 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 2024-12-04T09:45:01,225 WARN [IPC Server handler 3 on default port 33391 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-04T09:45:01,226 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 after 2ms 2024-12-04T09:45:02,076 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-04T09:45:02,076 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-04T09:45:02,077 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-04T09:45:02,078 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-04T09:45:03,817 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:53266 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33979:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53266 dst: /127.0.0.1:33979 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33979 remote=/127.0.0.1:53266]. Total timeout mills is 60000, 57379 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:03,818 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:42424 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42424 dst: /127.0.0.1:40851 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:03,818 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:38984 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:36275:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38984 dst: /127.0.0.1:36275 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:03,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741917_1101 (size=1368) 2024-12-04T09:45:03,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741917_1101 (size=1368) 2024-12-04T09:45:03,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741917_1101 (size=1368) 2024-12-04T09:45:05,100 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T09:45:05,227 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 after 4003ms 2024-12-04T09:45:05,229 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:05,230 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 took 4006ms 2024-12-04T09:45:05,231 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590; continuing. 2024-12-04T09:45:05,231 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 so closing down 2024-12-04T09:45:05,231 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:45:05,233 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733305500590.temp 2024-12-04T09:45:05,234 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000003-wal.1733305500590.temp 2024-12-04T09:45:05,234 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:45:05,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741923_1102 (size=635) 2024-12-04T09:45:05,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741923_1102 (size=635) 2024-12-04T09:45:05,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741923_1102 (size=635) 2024-12-04T09:45:05,241 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000003-wal.1733305500590.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T09:45:05,242 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000003-wal.1733305500590.temp to hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000008 2024-12-04T09:45:05,242 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T09:45:05,243 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590, journal: Splitting hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590, size=0 (0bytes) at 1733305501224Finishing writing output for hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 so closing down at 1733305505231 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000003-wal.1733305500590.temp at 1733305505234 (+3 ms)3 split writer threads finished at 1733305505234Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000003-wal.1733305500590.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733305505241 (+7 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000003-wal.1733305500590.temp to hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000008 at 1733305505242 (+1 ms)Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590, size=0, length=0, corrupted=false, cancelled=false at 1733305505242 2024-12-04T09:45:05,244 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305500590 2024-12-04T09:45:05,245 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000008 2024-12-04T09:45:05,245 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:45:05,247 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:05,257 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305505247, exclude list is [], retry=0 2024-12-04T09:45:05,259 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:05,260 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:05,260 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:05,261 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305505247 2024-12-04T09:45:05,262 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537)] 2024-12-04T09:45:05,262 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0c740b9030c1aa9c3838949a193179b, NAME => 'testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:45:05,262 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:05,262 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,262 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,264 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,265 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName a 2024-12-04T09:45:05,265 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,271 DEBUG [StoreFileOpener-d0c740b9030c1aa9c3838949a193179b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1162039237864fa5849b56d66c2bfba1_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T09:45:05,271 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_ 2024-12-04T09:45:05,275 DEBUG [StoreFileOpener-d0c740b9030c1aa9c3838949a193179b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T09:45:05,275 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_ 2024-12-04T09:45:05,278 DEBUG [StoreFileOpener-d0c740b9030c1aa9c3838949a193179b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4f656f2365a149ec987f8a2118005118_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T09:45:05,278 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_ 2024-12-04T09:45:05,282 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508 2024-12-04T09:45:05,286 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/d723107983ed44caa9dc61ad95d12a61 2024-12-04T09:45:05,286 WARN [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@362c4141 2024-12-04T09:45:05,286 WARN [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@362c4141 2024-12-04T09:45:05,286 WARN [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@362c4141 2024-12-04T09:45:05,286 WARN [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@362c4141 2024-12-04T09:45:05,286 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508] to archive 2024-12-04T09:45:05,287 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T09:45:05,289 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_ to hdfs://localhost:33391/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/1162039237864fa5849b56d66c2bfba1_SeqId_4_ 2024-12-04T09:45:05,290 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_ to hdfs://localhost:33391/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_ 2024-12-04T09:45:05,291 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_ to hdfs://localhost:33391/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/4f656f2365a149ec987f8a2118005118_SeqId_4_ 2024-12-04T09:45:05,292 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508 to hdfs://localhost:33391/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/a/bffd9ceb4e3a482ea84300c3a7de4508 2024-12-04T09:45:05,292 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,293 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,293 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName b 2024-12-04T09:45:05,293 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,294 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,294 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,295 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c740b9030c1aa9c3838949a193179b columnFamilyName c 2024-12-04T09:45:05,295 DEBUG [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,296 INFO [StoreOpener-d0c740b9030c1aa9c3838949a193179b-1 {}] regionserver.HStore(327): Store=d0c740b9030c1aa9c3838949a193179b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,296 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,297 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,298 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,299 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000008 2024-12-04T09:45:05,301 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:05,302 DEBUG [Time-limited test {}] regionserver.HRegion(5836): d0c740b9030c1aa9c3838949a193179b : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "d0c740b9030c1aa9c3838949a193179b" family_name: "a" compaction_input: "bffd9ceb4e3a482ea84300c3a7de4508" compaction_input: "1162039237864fa5849b56d66c2bfba1_SeqId_4_" compaction_input: "4f656f2365a149ec987f8a2118005118_SeqId_4_" compaction_input: "4653d5e0fb6d4fa2bd0917f6bbeb7609_SeqId_4_" compaction_output: "d723107983ed44caa9dc61ad95d12a61" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-04T09:45:05,302 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-04T09:45:05,303 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000008 2024-12-04T09:45:05,303 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/0000000000000000008 2024-12-04T09:45:05,304 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,304 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,305 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:45:05,306 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0c740b9030c1aa9c3838949a193179b 2024-12-04T09:45:05,308 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testCompactedBulkLoadedFiles/d0c740b9030c1aa9c3838949a193179b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T09:45:05,309 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0c740b9030c1aa9c3838949a193179b; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71654202, jitterRate=0.06773081421852112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:45:05,310 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0c740b9030c1aa9c3838949a193179b: Writing region info on filesystem at 1733305505262Initializing all the Stores at 1733305505263 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505263Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505263Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505263Cleaning up temporary data from old regions at 1733305505304 (+41 ms)Region opened successfully at 1733305505310 (+6 ms) 2024-12-04T09:45:05,312 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0c740b9030c1aa9c3838949a193179b, disabling compactions & flushes 2024-12-04T09:45:05,312 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:05,312 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:05,312 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. after waiting 0 ms 2024-12-04T09:45:05,312 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:05,313 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733305500495.d0c740b9030c1aa9c3838949a193179b. 2024-12-04T09:45:05,313 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0c740b9030c1aa9c3838949a193179b: Waiting for close lock at 1733305505312Disabling compacts and flushes for region at 1733305505312Disabling writes for close at 1733305505312Writing region close event to WAL at 1733305505313 (+1 ms)Closed at 1733305505313 2024-12-04T09:45:05,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741924_1103 (size=93) 2024-12-04T09:45:05,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741924_1103 (size=93) 2024-12-04T09:45:05,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741924_1103 (size=93) 2024-12-04T09:45:05,318 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:45:05,318 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733305505247) 2024-12-04T09:45:05,331 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=438 (was 423) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:33391 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33391 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-717161987_22 at /127.0.0.1:53356 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-717161987_22 at /127.0.0.1:39050 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-717161987_22 at /127.0.0.1:42512 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=351 (was 365), ProcessCount=11 (was 11), AvailableMemoryMB=10113 (was 10392) 2024-12-04T09:45:05,331 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-04T09:45:05,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=351, ProcessCount=11, AvailableMemoryMB=10112 2024-12-04T09:45:05,342 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-04T09:45:05,355 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:45:05,356 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T09:45:05,357 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T09:45:05,359 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-29313020, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/hregion-29313020, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:05,371 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-29313020/hregion-29313020.1733305505359, exclude list is [], retry=0 2024-12-04T09:45:05,374 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:05,374 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:05,374 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:05,376 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-29313020/hregion-29313020.1733305505359 2024-12-04T09:45:05,376 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:45:05,376 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => de01042e1873405fa0c5ddb850cdde85, NAME => 'testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33391/hbase 2024-12-04T09:45:05,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741926_1105 (size=67) 2024-12-04T09:45:05,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741926_1105 (size=67) 2024-12-04T09:45:05,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741926_1105 (size=67) 2024-12-04T09:45:05,384 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:05,385 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,386 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName a 2024-12-04T09:45:05,387 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,387 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,387 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,388 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName b 2024-12-04T09:45:05,389 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,389 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,389 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,390 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName c 2024-12-04T09:45:05,390 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,391 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,391 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,392 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,392 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,393 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,393 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,394 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:45:05,395 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,398 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T09:45:05,398 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de01042e1873405fa0c5ddb850cdde85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60289554, jitterRate=-0.10161563754081726}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:45:05,398 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de01042e1873405fa0c5ddb850cdde85: Writing region info on filesystem at 1733305505384Initializing all the Stores at 1733305505385 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505385Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505385Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505385Cleaning up temporary data from old regions at 1733305505393 (+8 ms)Region opened successfully at 1733305505398 (+5 ms) 2024-12-04T09:45:05,399 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de01042e1873405fa0c5ddb850cdde85, disabling compactions & flushes 2024-12-04T09:45:05,399 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,399 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,399 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. after waiting 0 ms 2024-12-04T09:45:05,399 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,399 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,399 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de01042e1873405fa0c5ddb850cdde85: Waiting for close lock at 1733305505399Disabling compacts and flushes for region at 1733305505399Disabling writes for close at 1733305505399Writing region close event to WAL at 1733305505399Closed at 1733305505399 2024-12-04T09:45:05,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741925_1104 (size=93) 2024-12-04T09:45:05,402 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-29313020/hregion-29313020.1733305505359 not finished, retry = 0 2024-12-04T09:45:05,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741925_1104 (size=93) 2024-12-04T09:45:05,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741925_1104 (size=93) 2024-12-04T09:45:05,505 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:45:05,505 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-29313020:(num 1733305505359) 2024-12-04T09:45:05,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:45:05,507 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:05,518 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507, exclude list is [], retry=0 2024-12-04T09:45:05,520 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:05,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:05,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:05,522 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507 2024-12-04T09:45:05,522 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:45:05,523 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de01042e1873405fa0c5ddb850cdde85, NAME => 'testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:45:05,523 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:05,523 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,523 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,524 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,525 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName a 2024-12-04T09:45:05,525 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,526 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,526 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,527 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName b 2024-12-04T09:45:05,527 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,527 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,527 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,528 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName c 2024-12-04T09:45:05,528 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,529 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,529 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,530 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,532 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,532 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,533 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:45:05,534 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,535 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de01042e1873405fa0c5ddb850cdde85; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69410534, jitterRate=0.034297555685043335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:45:05,535 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de01042e1873405fa0c5ddb850cdde85: Writing region info on filesystem at 1733305505523Initializing all the Stores at 1733305505524 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505524Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505524Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505524Cleaning up temporary data from old regions at 1733305505532 (+8 ms)Region opened successfully at 1733305505535 (+3 ms) 2024-12-04T09:45:05,543 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing de01042e1873405fa0c5ddb850cdde85 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T09:45:05,559 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/a/03b972bdc536406185d3397a796c776b is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733305505535/Put/seqid=0 2024-12-04T09:45:05,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741928_1107 (size=5958) 2024-12-04T09:45:05,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741928_1107 (size=5958) 2024-12-04T09:45:05,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741928_1107 (size=5958) 2024-12-04T09:45:05,565 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/a/03b972bdc536406185d3397a796c776b 2024-12-04T09:45:05,570 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/a/03b972bdc536406185d3397a796c776b as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/a/03b972bdc536406185d3397a796c776b 2024-12-04T09:45:05,576 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/a/03b972bdc536406185d3397a796c776b, entries=10, sequenceid=13, filesize=5.8 K 2024-12-04T09:45:05,577 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for de01042e1873405fa0c5ddb850cdde85 in 34ms, sequenceid=13, compaction requested=false 2024-12-04T09:45:05,577 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for de01042e1873405fa0c5ddb850cdde85: 2024-12-04T09:45:05,595 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de01042e1873405fa0c5ddb850cdde85, disabling compactions & flushes 2024-12-04T09:45:05,595 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,595 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,595 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. after waiting 0 ms 2024-12-04T09:45:05,595 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,596 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,596 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:05,596 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de01042e1873405fa0c5ddb850cdde85: Waiting for close lock at 1733305505595Disabling compacts and flushes for region at 1733305505595Disabling writes for close at 1733305505595Writing region close event to WAL at 1733305505596 (+1 ms)Closed at 1733305505596 2024-12-04T09:45:05,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741927_1106 (size=2805) 2024-12-04T09:45:05,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741927_1106 (size=2805) 2024-12-04T09:45:05,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741927_1106 (size=2805) 2024-12-04T09:45:05,613 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507, size=2.7 K (2805bytes) 2024-12-04T09:45:05,613 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507 2024-12-04T09:45:05,613 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507 after 0ms 2024-12-04T09:45:05,615 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:05,616 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507 took 3ms 2024-12-04T09:45:05,618 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507 so closing down 2024-12-04T09:45:05,618 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:45:05,619 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733305505507.temp 2024-12-04T09:45:05,620 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000003-wal.1733305505507.temp 2024-12-04T09:45:05,620 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:45:05,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741929_1108 (size=2312) 2024-12-04T09:45:05,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741929_1108 (size=2312) 2024-12-04T09:45:05,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741929_1108 (size=2312) 2024-12-04T09:45:05,627 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000003-wal.1733305505507.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T09:45:05,628 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000003-wal.1733305505507.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000035 2024-12-04T09:45:05,628 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-04T09:45:05,628 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507, journal: Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507, size=2.7 K (2805bytes) at 1733305505613Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507 so closing down at 1733305505618 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000003-wal.1733305505507.temp at 1733305505620 (+2 ms)3 split writer threads finished at 1733305505620Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000003-wal.1733305505507.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733305505627 (+7 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000003-wal.1733305505507.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000035 at 1733305505628 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733305505628 2024-12-04T09:45:05,630 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505507 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305505507 2024-12-04T09:45:05,631 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000035 2024-12-04T09:45:05,631 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:45:05,633 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:05,644 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633, exclude list is [], retry=0 2024-12-04T09:45:05,646 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:05,646 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:05,646 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:05,648 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 2024-12-04T09:45:05,648 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799),(127.0.0.1/127.0.0.1:44681:44681)] 2024-12-04T09:45:05,649 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de01042e1873405fa0c5ddb850cdde85, NAME => 'testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85.', STARTKEY => '', ENDKEY => ''} 2024-12-04T09:45:05,649 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:05,649 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,649 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,650 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,651 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName a 2024-12-04T09:45:05,651 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,656 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/a/03b972bdc536406185d3397a796c776b 2024-12-04T09:45:05,656 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,656 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,657 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName b 2024-12-04T09:45:05,657 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,658 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,658 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,659 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName c 2024-12-04T09:45:05,659 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:05,659 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:05,659 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,660 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,661 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,661 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000035 2024-12-04T09:45:05,663 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:05,663 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000035 2024-12-04T09:45:05,664 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing de01042e1873405fa0c5ddb850cdde85 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-04T09:45:05,676 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/b/a5e0cd215bea470091ee4c1356f1c85a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733305505577/Put/seqid=0 2024-12-04T09:45:05,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741931_1110 (size=5958) 2024-12-04T09:45:05,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741931_1110 (size=5958) 2024-12-04T09:45:05,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741931_1110 (size=5958) 2024-12-04T09:45:05,682 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/b/a5e0cd215bea470091ee4c1356f1c85a 2024-12-04T09:45:05,707 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/c/4f112fe908fe4240bc989bbebea800f0 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733305505585/Put/seqid=0 2024-12-04T09:45:05,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741932_1111 (size=5958) 2024-12-04T09:45:05,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741932_1111 (size=5958) 2024-12-04T09:45:05,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741932_1111 (size=5958) 2024-12-04T09:45:05,714 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/c/4f112fe908fe4240bc989bbebea800f0 2024-12-04T09:45:05,720 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/b/a5e0cd215bea470091ee4c1356f1c85a as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/b/a5e0cd215bea470091ee4c1356f1c85a 2024-12-04T09:45:05,725 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/b/a5e0cd215bea470091ee4c1356f1c85a, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T09:45:05,726 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/c/4f112fe908fe4240bc989bbebea800f0 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/c/4f112fe908fe4240bc989bbebea800f0 2024-12-04T09:45:05,732 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/c/4f112fe908fe4240bc989bbebea800f0, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T09:45:05,732 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for de01042e1873405fa0c5ddb850cdde85 in 68ms, sequenceid=35, compaction requested=false; wal=null 2024-12-04T09:45:05,733 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000035 2024-12-04T09:45:05,734 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,734 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,735 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:45:05,736 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:05,738 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-04T09:45:05,739 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de01042e1873405fa0c5ddb850cdde85; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59771804, jitterRate=-0.10933071374893188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:45:05,739 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de01042e1873405fa0c5ddb850cdde85: Writing region info on filesystem at 1733305505649Initializing all the Stores at 1733305505650 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505650Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505650Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305505650Obtaining lock to block concurrent updates at 1733305505664 (+14 ms)Preparing flush snapshotting stores in de01042e1873405fa0c5ddb850cdde85 at 1733305505664Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733305505664Flushing stores of testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. at 1733305505664Flushing de01042e1873405fa0c5ddb850cdde85/b: creating writer at 1733305505664Flushing de01042e1873405fa0c5ddb850cdde85/b: appending metadata at 1733305505675 (+11 ms)Flushing de01042e1873405fa0c5ddb850cdde85/b: closing flushed file at 1733305505675Flushing de01042e1873405fa0c5ddb850cdde85/c: creating writer at 1733305505687 (+12 ms)Flushing de01042e1873405fa0c5ddb850cdde85/c: appending metadata at 1733305505706 (+19 ms)Flushing de01042e1873405fa0c5ddb850cdde85/c: closing flushed file at 1733305505706Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7476dfe4: reopening flushed file at 1733305505719 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70cd8acf: reopening flushed file at 1733305505725 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for de01042e1873405fa0c5ddb850cdde85 in 68ms, sequenceid=35, compaction requested=false; wal=null at 1733305505732 (+7 ms)Cleaning up temporary data from old regions at 1733305505734 (+2 ms)Region opened successfully at 1733305505739 (+5 ms) 2024-12-04T09:45:05,802 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633, size=0 (0bytes) 2024-12-04T09:45:05,802 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 might be still open, length is 0 2024-12-04T09:45:05,802 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 2024-12-04T09:45:05,802 WARN [IPC Server handler 1 on default port 33391 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-04T09:45:05,803 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 after 1ms 2024-12-04T09:45:06,815 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:39140 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:36275:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39140 dst: /127.0.0.1:36275 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36275 remote=/127.0.0.1:39140]. Total timeout mills is 60000, 58954 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:06,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:42556 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40851:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42556 dst: /127.0.0.1:40851 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:06,816 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2050232587_22 at /127.0.0.1:53428 [Receiving block BP-129416318-172.17.0.2-1733305468255:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33979:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53428 dst: /127.0.0.1:33979 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:06,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741930_1112 (size=2304) 2024-12-04T09:45:06,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741930_1112 (size=2304) 2024-12-04T09:45:09,804 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 after 4002ms 2024-12-04T09:45:09,852 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:09,852 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 took 4051ms 2024-12-04T09:45:09,856 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633; continuing. 2024-12-04T09:45:09,856 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 so closing down 2024-12-04T09:45:09,856 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T09:45:09,859 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733305505633.temp 2024-12-04T09:45:09,861 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000037-wal.1733305505633.temp 2024-12-04T09:45:09,861 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T09:45:09,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741933_1113 (size=2312) 2024-12-04T09:45:09,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741933_1113 (size=2312) 2024-12-04T09:45:09,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741933_1113 (size=2312) 2024-12-04T09:45:09,871 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000037-wal.1733305505633.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T09:45:09,872 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000037-wal.1733305505633.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000066 2024-12-04T09:45:09,872 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T09:45:09,872 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633, journal: Splitting hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633, size=0 (0bytes) at 1733305505802Finishing writing output for hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 so closing down at 1733305509856 (+4054 ms)Creating recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000037-wal.1733305505633.temp at 1733305509861 (+5 ms)3 split writer threads finished at 1733305509861Closed recovered edits writer path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000037-wal.1733305505633.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733305509871 (+10 ms)Rename recovered edits hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000037-wal.1733305505633.temp to hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000066 at 1733305509872 (+1 ms)Processed 30 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633, size=0, length=0, corrupted=false, cancelled=false at 1733305509872 2024-12-04T09:45:09,873 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 to hdfs://localhost:33391/hbase/oldWALs/wal.1733305505633 2024-12-04T09:45:09,874 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000066 2024-12-04T09:45:09,874 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T09:45:09,876 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33391/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354, archiveDir=hdfs://localhost:33391/hbase/oldWALs, maxLogs=32 2024-12-04T09:45:09,891 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305509876, exclude list is [], retry=0 2024-12-04T09:45:09,893 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40851,DS-05b3786b-0c64-46eb-bf82-3152ca69c7e6,DISK] 2024-12-04T09:45:09,894 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36275,DS-ee3ed8ce-078d-4e16-accd-040556b684ca,DISK] 2024-12-04T09:45:09,894 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33979,DS-babedffc-bc4a-4b57-b95f-2ff7d45158e0,DISK] 2024-12-04T09:45:09,898 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305509876 2024-12-04T09:45:09,899 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44681:44681),(127.0.0.1/127.0.0.1:43537:43537),(127.0.0.1/127.0.0.1:35799:35799)] 2024-12-04T09:45:09,899 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T09:45:09,902 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:09,904 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName a 2024-12-04T09:45:09,904 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:09,910 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/a/03b972bdc536406185d3397a796c776b 2024-12-04T09:45:09,910 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:09,911 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:09,911 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName b 2024-12-04T09:45:09,911 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:09,918 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/b/a5e0cd215bea470091ee4c1356f1c85a 2024-12-04T09:45:09,918 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:09,918 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:09,919 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de01042e1873405fa0c5ddb850cdde85 columnFamilyName c 2024-12-04T09:45:09,919 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T09:45:09,924 DEBUG [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/c/4f112fe908fe4240bc989bbebea800f0 2024-12-04T09:45:09,924 INFO [StoreOpener-de01042e1873405fa0c5ddb850cdde85-1 {}] regionserver.HStore(327): Store=de01042e1873405fa0c5ddb850cdde85/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T09:45:09,924 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:09,925 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:09,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:09,927 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000066 2024-12-04T09:45:09,928 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-04T09:45:09,931 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000066 2024-12-04T09:45:09,932 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing de01042e1873405fa0c5ddb850cdde85 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T09:45:09,944 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/a/7d0e9ea08add4c14bea9ec670e10cc4b is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733305505746/Put/seqid=0 2024-12-04T09:45:09,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741935_1115 (size=5958) 2024-12-04T09:45:09,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741935_1115 (size=5958) 2024-12-04T09:45:09,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741935_1115 (size=5958) 2024-12-04T09:45:09,950 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/a/7d0e9ea08add4c14bea9ec670e10cc4b 2024-12-04T09:45:09,967 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/b/75a265c1fcf14de09b3808b72af185aa is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733305505754/Put/seqid=0 2024-12-04T09:45:09,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741936_1116 (size=5958) 2024-12-04T09:45:09,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741936_1116 (size=5958) 2024-12-04T09:45:09,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741936_1116 (size=5958) 2024-12-04T09:45:09,974 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/b/75a265c1fcf14de09b3808b72af185aa 2024-12-04T09:45:09,991 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/c/16a2ecaccf834b9581d5866d5e8089b2 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733305505764/Put/seqid=0 2024-12-04T09:45:09,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741937_1117 (size=5958) 2024-12-04T09:45:09,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741937_1117 (size=5958) 2024-12-04T09:45:09,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741937_1117 (size=5958) 2024-12-04T09:45:09,997 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/c/16a2ecaccf834b9581d5866d5e8089b2 2024-12-04T09:45:10,002 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/a/7d0e9ea08add4c14bea9ec670e10cc4b as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/a/7d0e9ea08add4c14bea9ec670e10cc4b 2024-12-04T09:45:10,007 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/a/7d0e9ea08add4c14bea9ec670e10cc4b, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T09:45:10,009 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/b/75a265c1fcf14de09b3808b72af185aa as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/b/75a265c1fcf14de09b3808b72af185aa 2024-12-04T09:45:10,014 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/b/75a265c1fcf14de09b3808b72af185aa, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T09:45:10,015 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/.tmp/c/16a2ecaccf834b9581d5866d5e8089b2 as hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/c/16a2ecaccf834b9581d5866d5e8089b2 2024-12-04T09:45:10,019 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/c/16a2ecaccf834b9581d5866d5e8089b2, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T09:45:10,020 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for de01042e1873405fa0c5ddb850cdde85 in 89ms, sequenceid=66, compaction requested=false; wal=null 2024-12-04T09:45:10,020 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/0000000000000000066 2024-12-04T09:45:10,021 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:10,022 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:10,022 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T09:45:10,024 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de01042e1873405fa0c5ddb850cdde85 2024-12-04T09:45:10,026 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/hbase/data/default/testReplayEditsWrittenViaHRegion/de01042e1873405fa0c5ddb850cdde85/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-04T09:45:10,027 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de01042e1873405fa0c5ddb850cdde85; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61500137, jitterRate=-0.08357654511928558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T09:45:10,028 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de01042e1873405fa0c5ddb850cdde85: Writing region info on filesystem at 1733305509899Initializing all the Stores at 1733305509900 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305509900Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305509902 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733305509902Obtaining lock to block concurrent updates at 1733305509932 (+30 ms)Preparing flush snapshotting stores in de01042e1873405fa0c5ddb850cdde85 at 1733305509932Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733305509932Flushing stores of testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. at 1733305509932Flushing de01042e1873405fa0c5ddb850cdde85/a: creating writer at 1733305509932Flushing de01042e1873405fa0c5ddb850cdde85/a: appending metadata at 1733305509943 (+11 ms)Flushing de01042e1873405fa0c5ddb850cdde85/a: closing flushed file at 1733305509943Flushing de01042e1873405fa0c5ddb850cdde85/b: creating writer at 1733305509954 (+11 ms)Flushing de01042e1873405fa0c5ddb850cdde85/b: appending metadata at 1733305509967 (+13 ms)Flushing de01042e1873405fa0c5ddb850cdde85/b: closing flushed file at 1733305509967Flushing de01042e1873405fa0c5ddb850cdde85/c: creating writer at 1733305509978 (+11 ms)Flushing de01042e1873405fa0c5ddb850cdde85/c: appending metadata at 1733305509991 (+13 ms)Flushing de01042e1873405fa0c5ddb850cdde85/c: closing flushed file at 1733305509991Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6646a046: reopening flushed file at 1733305510001 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b36f68e: reopening flushed file at 1733305510008 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4110c741: reopening flushed file at 1733305510014 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for de01042e1873405fa0c5ddb850cdde85 in 89ms, sequenceid=66, compaction requested=false; wal=null at 1733305510020 (+6 ms)Cleaning up temporary data from old regions at 1733305510022 (+2 ms)Region opened successfully at 1733305510028 (+6 ms) 2024-12-04T09:45:10,040 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de01042e1873405fa0c5ddb850cdde85, disabling compactions & flushes 2024-12-04T09:45:10,040 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:10,040 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:10,040 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. after waiting 0 ms 2024-12-04T09:45:10,040 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:10,042 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733305505355.de01042e1873405fa0c5ddb850cdde85. 2024-12-04T09:45:10,042 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de01042e1873405fa0c5ddb850cdde85: Waiting for close lock at 1733305510040Disabling compacts and flushes for region at 1733305510040Disabling writes for close at 1733305510040Writing region close event to WAL at 1733305510041 (+1 ms)Closed at 1733305510041 2024-12-04T09:45:10,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741934_1114 (size=93) 2024-12-04T09:45:10,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741934_1114 (size=93) 2024-12-04T09:45:10,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741934_1114 (size=93) 2024-12-04T09:45:10,046 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T09:45:10,046 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733305509876) 2024-12-04T09:45:10,061 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=443 (was 438) Potentially hanging thread: IPC Client (102277669) connection to localhost/127.0.0.1:33391 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33391 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1994327813_22 at /127.0.0.1:46236 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1994327813_22 at /127.0.0.1:55550 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1994327813_22 at /127.0.0.1:45880 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=323 (was 351), ProcessCount=11 (was 11), AvailableMemoryMB=10094 (was 10112) 2024-12-04T09:45:10,061 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-04T09:45:10,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T09:45:10,061 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T09:45:10,061 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T09:45:10,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:45:10,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:45:10,062 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T09:45:10,062 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T09:45:10,062 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=269611031, stopped=false 2024-12-04T09:45:10,063 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=84486a41f81c,38097,1733305471718 2024-12-04T09:45:10,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T09:45:10,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T09:45:10,079 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T09:45:10,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:45:10,079 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:45:10,079 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:45:10,079 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T09:45:10,080 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T09:45:10,080 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T09:45:10,080 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T09:45:10,080 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:45:10,080 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T09:45:10,080 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,35147,1733305472454' ***** 2024-12-04T09:45:10,080 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T09:45:10,080 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '84486a41f81c,36773,1733305472592' ***** 2024-12-04T09:45:10,080 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T09:45:10,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T09:45:10,081 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T09:45:10,081 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T09:45:10,081 INFO [RS:2;84486a41f81c:36773 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T09:45:10,081 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T09:45:10,081 INFO [RS:0;84486a41f81c:35147 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T09:45:10,081 INFO [RS:2;84486a41f81c:36773 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T09:45:10,081 INFO [RS:0;84486a41f81c:35147 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T09:45:10,081 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T09:45:10,081 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(959): stopping server 84486a41f81c,36773,1733305472592 2024-12-04T09:45:10,081 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(3091): Received CLOSE for 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:45:10,081 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(959): stopping server 84486a41f81c,35147,1733305472454 2024-12-04T09:45:10,081 INFO [RS:0;84486a41f81c:35147 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T09:45:10,081 INFO [RS:0;84486a41f81c:35147 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;84486a41f81c:35147. 2024-12-04T09:45:10,081 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 56ae12cc64f9a720b1c872193297ac28, disabling compactions & flushes 2024-12-04T09:45:10,082 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:45:10,082 DEBUG [RS:0;84486a41f81c:35147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T09:45:10,082 DEBUG [RS:0;84486a41f81c:35147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:45:10,082 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:45:10,082 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. after waiting 0 ms 2024-12-04T09:45:10,082 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T09:45:10,082 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:45:10,082 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1325): Online Regions={56ae12cc64f9a720b1c872193297ac28=testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28.} 2024-12-04T09:45:10,082 DEBUG [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1351): Waiting on 56ae12cc64f9a720b1c872193297ac28 2024-12-04T09:45:10,082 INFO [RS:2;84486a41f81c:36773 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T09:45:10,082 INFO [RS:2;84486a41f81c:36773 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;84486a41f81c:36773. 2024-12-04T09:45:10,082 DEBUG [RS:2;84486a41f81c:36773 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T09:45:10,082 DEBUG [RS:2;84486a41f81c:36773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:45:10,082 INFO [RS:2;84486a41f81c:36773 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T09:45:10,082 INFO [RS:2;84486a41f81c:36773 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T09:45:10,082 INFO [RS:2;84486a41f81c:36773 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T09:45:10,082 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T09:45:10,083 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T09:45:10,083 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T09:45:10,083 DEBUG [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T09:45:10,083 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T09:45:10,083 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T09:45:10,083 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T09:45:10,083 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T09:45:10,083 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T09:45:10,083 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-04T09:45:10,087 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/default/testReplayEditsAfterRegionMovedWithMultiCF/56ae12cc64f9a720b1c872193297ac28/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-04T09:45:10,087 INFO [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:45:10,087 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 56ae12cc64f9a720b1c872193297ac28: Waiting for close lock at 1733305510081Running coprocessor pre-close hooks at 1733305510081Disabling compacts and flushes for region at 1733305510081Disabling writes for close at 1733305510082 (+1 ms)Writing region close event to WAL at 1733305510083 (+1 ms)Running coprocessor post-close hooks at 1733305510087 (+4 ms)Closed at 1733305510087 2024-12-04T09:45:10,087 DEBUG [RS_CLOSE_REGION-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28. 2024-12-04T09:45:10,102 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T09:45:10,102 INFO [regionserver/84486a41f81c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T09:45:10,109 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/info/13e03dae3dec4580800c38e7da5d576f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733305486449.56ae12cc64f9a720b1c872193297ac28./info:regioninfo/1733305489623/Put/seqid=0 2024-12-04T09:45:10,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741938_1118 (size=8243) 2024-12-04T09:45:10,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741938_1118 (size=8243) 2024-12-04T09:45:10,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741938_1118 (size=8243) 2024-12-04T09:45:10,116 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/info/13e03dae3dec4580800c38e7da5d576f 2024-12-04T09:45:10,136 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/ns/4c3a42a28fa84d3bac24230d08bf8c5d is 43, key is default/ns:d/1733305474644/Put/seqid=0 2024-12-04T09:45:10,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741939_1119 (size=5153) 2024-12-04T09:45:10,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741939_1119 (size=5153) 2024-12-04T09:45:10,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741939_1119 (size=5153) 2024-12-04T09:45:10,143 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/ns/4c3a42a28fa84d3bac24230d08bf8c5d 2024-12-04T09:45:10,160 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/table/1334784dbf9442439354c2713c4d7003 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733305486875/Put/seqid=0 2024-12-04T09:45:10,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741940_1120 (size=5431) 2024-12-04T09:45:10,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741940_1120 (size=5431) 2024-12-04T09:45:10,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741940_1120 (size=5431) 2024-12-04T09:45:10,167 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/table/1334784dbf9442439354c2713c4d7003 2024-12-04T09:45:10,172 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/info/13e03dae3dec4580800c38e7da5d576f as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/info/13e03dae3dec4580800c38e7da5d576f 2024-12-04T09:45:10,177 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/info/13e03dae3dec4580800c38e7da5d576f, entries=18, sequenceid=21, filesize=8.0 K 2024-12-04T09:45:10,178 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/ns/4c3a42a28fa84d3bac24230d08bf8c5d as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/ns/4c3a42a28fa84d3bac24230d08bf8c5d 2024-12-04T09:45:10,184 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/ns/4c3a42a28fa84d3bac24230d08bf8c5d, entries=2, sequenceid=21, filesize=5.0 K 2024-12-04T09:45:10,185 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/.tmp/table/1334784dbf9442439354c2713c4d7003 as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/table/1334784dbf9442439354c2713c4d7003 2024-12-04T09:45:10,190 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/table/1334784dbf9442439354c2713c4d7003, entries=2, sequenceid=21, filesize=5.3 K 2024-12-04T09:45:10,191 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=21, compaction requested=false 2024-12-04T09:45:10,196 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-04T09:45:10,196 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T09:45:10,196 INFO [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T09:45:10,197 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733305510083Running coprocessor pre-close hooks at 1733305510083Disabling compacts and flushes for region at 1733305510083Disabling writes for close at 1733305510083Obtaining lock to block concurrent updates at 1733305510083Preparing flush snapshotting stores in 1588230740 at 1733305510083Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733305510084 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733305510084Flushing 1588230740/info: creating writer at 1733305510085 (+1 ms)Flushing 1588230740/info: appending metadata at 1733305510109 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733305510109Flushing 1588230740/ns: creating writer at 1733305510122 (+13 ms)Flushing 1588230740/ns: appending metadata at 1733305510136 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733305510136Flushing 1588230740/table: creating writer at 1733305510148 (+12 ms)Flushing 1588230740/table: appending metadata at 1733305510160 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733305510160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d5954b: reopening flushed file at 1733305510171 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e2ff6ec: reopening flushed file at 1733305510178 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a398ec9: reopening flushed file at 1733305510184 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 108ms, sequenceid=21, compaction requested=false at 1733305510191 (+7 ms)Writing region close event to WAL at 1733305510193 (+2 ms)Running coprocessor post-close hooks at 1733305510196 (+3 ms)Closed at 1733305510196 2024-12-04T09:45:10,197 DEBUG [RS_CLOSE_META-regionserver/84486a41f81c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T09:45:10,282 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,35147,1733305472454; all regions closed. 2024-12-04T09:45:10,283 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(976): stopping server 84486a41f81c,36773,1733305472592; all regions closed. 2024-12-04T09:45:10,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741833_1009 (size=2652) 2024-12-04T09:45:10,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741833_1009 (size=2652) 2024-12-04T09:45:10,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741833_1009 (size=2652) 2024-12-04T09:45:10,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741836_1012 (size=3783) 2024-12-04T09:45:10,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741836_1012 (size=3783) 2024-12-04T09:45:10,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741836_1012 (size=3783) 2024-12-04T09:45:10,293 DEBUG [RS:0;84486a41f81c:35147 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C35147%2C1733305472454:(num 1733305474029) 2024-12-04T09:45:10,294 DEBUG [RS:0;84486a41f81c:35147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T09:45:10,294 DEBUG [RS:2;84486a41f81c:36773 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs 2024-12-04T09:45:10,294 INFO [RS:2;84486a41f81c:36773 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C36773%2C1733305472592.meta:.meta(num 1733305474465) 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T09:45:10,294 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T09:45:10,294 INFO [RS:0;84486a41f81c:35147 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T09:45:10,295 INFO [RS:0;84486a41f81c:35147 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35147 2024-12-04T09:45:10,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741834_1010 (size=93) 2024-12-04T09:45:10,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741834_1010 (size=93) 2024-12-04T09:45:10,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741834_1010 (size=93) 2024-12-04T09:45:10,298 DEBUG [RS:2;84486a41f81c:36773 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/oldWALs 2024-12-04T09:45:10,298 INFO [RS:2;84486a41f81c:36773 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 84486a41f81c%2C36773%2C1733305472592:(num 1733305474029) 2024-12-04T09:45:10,298 DEBUG [RS:2;84486a41f81c:36773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T09:45:10,298 INFO [RS:2;84486a41f81c:36773 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T09:45:10,298 INFO [RS:2;84486a41f81c:36773 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T09:45:10,299 INFO [RS:2;84486a41f81c:36773 {}] hbase.ChoreService(370): Chore service for: regionserver/84486a41f81c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T09:45:10,299 INFO [RS:2;84486a41f81c:36773 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T09:45:10,299 INFO [regionserver/84486a41f81c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T09:45:10,299 INFO [RS:2;84486a41f81c:36773 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36773 2024-12-04T09:45:10,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,35147,1733305472454 2024-12-04T09:45:10,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T09:45:10,304 INFO [RS:0;84486a41f81c:35147 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T09:45:10,312 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/84486a41f81c,36773,1733305472592 2024-12-04T09:45:10,312 INFO [RS:2;84486a41f81c:36773 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T09:45:10,320 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,35147,1733305472454] 2024-12-04T09:45:10,337 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,35147,1733305472454 already deleted, retry=false 2024-12-04T09:45:10,337 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 84486a41f81c,35147,1733305472454 expired; onlineServers=1 2024-12-04T09:45:10,337 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [84486a41f81c,36773,1733305472592] 2024-12-04T09:45:10,345 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/84486a41f81c,36773,1733305472592 already deleted, retry=false 2024-12-04T09:45:10,346 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 84486a41f81c,36773,1733305472592 expired; onlineServers=0 2024-12-04T09:45:10,346 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '84486a41f81c,38097,1733305471718' ***** 2024-12-04T09:45:10,346 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T09:45:10,346 INFO [M:0;84486a41f81c:38097 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T09:45:10,346 INFO [M:0;84486a41f81c:38097 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T09:45:10,347 DEBUG [M:0;84486a41f81c:38097 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T09:45:10,347 DEBUG [M:0;84486a41f81c:38097 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T09:45:10,347 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T09:45:10,347 DEBUG [master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733305473687 {}] cleaner.HFileCleaner(306): Exit Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.small.0-1733305473687,5,FailOnTimeoutGroup] 2024-12-04T09:45:10,347 DEBUG [master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733305473681 {}] cleaner.HFileCleaner(306): Exit Thread[master/84486a41f81c:0:becomeActiveMaster-HFileCleaner.large.0-1733305473681,5,FailOnTimeoutGroup] 2024-12-04T09:45:10,347 INFO [M:0;84486a41f81c:38097 {}] hbase.ChoreService(370): Chore service for: master/84486a41f81c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T09:45:10,348 INFO [M:0;84486a41f81c:38097 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T09:45:10,348 DEBUG [M:0;84486a41f81c:38097 {}] master.HMaster(1795): Stopping service threads 2024-12-04T09:45:10,348 INFO [M:0;84486a41f81c:38097 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T09:45:10,348 INFO [M:0;84486a41f81c:38097 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T09:45:10,349 INFO [M:0;84486a41f81c:38097 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T09:45:10,350 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T09:45:10,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T09:45:10,354 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T09:45:10,354 DEBUG [M:0;84486a41f81c:38097 {}] zookeeper.ZKUtil(347): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T09:45:10,354 WARN [M:0;84486a41f81c:38097 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T09:45:10,355 INFO [M:0;84486a41f81c:38097 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/.lastflushedseqids 2024-12-04T09:45:10,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741941_1121 (size=138) 2024-12-04T09:45:10,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741941_1121 (size=138) 2024-12-04T09:45:10,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741941_1121 (size=138) 2024-12-04T09:45:10,372 INFO [M:0;84486a41f81c:38097 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T09:45:10,372 INFO [M:0;84486a41f81c:38097 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T09:45:10,372 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T09:45:10,372 INFO [M:0;84486a41f81c:38097 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:45:10,372 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:45:10,372 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T09:45:10,372 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:45:10,372 INFO [M:0;84486a41f81c:38097 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-12-04T09:45:10,386 DEBUG [M:0;84486a41f81c:38097 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cadce20bc4e14d13a7cfabd01e908244 is 82, key is hbase:meta,,1/info:regioninfo/1733305474544/Put/seqid=0 2024-12-04T09:45:10,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741942_1122 (size=5672) 2024-12-04T09:45:10,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741942_1122 (size=5672) 2024-12-04T09:45:10,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741942_1122 (size=5672) 2024-12-04T09:45:10,392 INFO [M:0;84486a41f81c:38097 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cadce20bc4e14d13a7cfabd01e908244 2024-12-04T09:45:10,412 DEBUG [M:0;84486a41f81c:38097 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7f96cfe0d506497391c6c6c9a7d10e6e is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733305486880/Put/seqid=0 2024-12-04T09:45:10,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741943_1123 (size=7755) 2024-12-04T09:45:10,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741943_1123 (size=7755) 2024-12-04T09:45:10,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741943_1123 (size=7755) 2024-12-04T09:45:10,420 INFO [M:0;84486a41f81c:38097 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7f96cfe0d506497391c6c6c9a7d10e6e 2024-12-04T09:45:10,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:45:10,421 INFO [RS:0;84486a41f81c:35147 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T09:45:10,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35147-0x101a10432830001, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:45:10,421 INFO [RS:0;84486a41f81c:35147 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,35147,1733305472454; zookeeper connection closed. 2024-12-04T09:45:10,421 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4589c19f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4589c19f 2024-12-04T09:45:10,424 INFO [M:0;84486a41f81c:38097 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7f96cfe0d506497391c6c6c9a7d10e6e 2024-12-04T09:45:10,429 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:45:10,429 INFO [RS:2;84486a41f81c:36773 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T09:45:10,429 DEBUG [pool-76-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36773-0x101a10432830003, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:45:10,429 INFO [RS:2;84486a41f81c:36773 {}] regionserver.HRegionServer(1031): Exiting; stopping=84486a41f81c,36773,1733305472592; zookeeper connection closed. 2024-12-04T09:45:10,429 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@423fd9b9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@423fd9b9 2024-12-04T09:45:10,430 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-04T09:45:10,438 DEBUG [M:0;84486a41f81c:38097 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d5d9cef2cfd243329bade5c07bdb2575 is 69, key is 84486a41f81c,35147,1733305472454/rs:state/1733305473763/Put/seqid=0 2024-12-04T09:45:10,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741944_1124 (size=5440) 2024-12-04T09:45:10,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741944_1124 (size=5440) 2024-12-04T09:45:10,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741944_1124 (size=5440) 2024-12-04T09:45:10,449 INFO [M:0;84486a41f81c:38097 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d5d9cef2cfd243329bade5c07bdb2575 2024-12-04T09:45:10,454 INFO [M:0;84486a41f81c:38097 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d5d9cef2cfd243329bade5c07bdb2575 2024-12-04T09:45:10,455 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cadce20bc4e14d13a7cfabd01e908244 as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cadce20bc4e14d13a7cfabd01e908244 2024-12-04T09:45:10,461 INFO [M:0;84486a41f81c:38097 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cadce20bc4e14d13a7cfabd01e908244, entries=8, sequenceid=168, filesize=5.5 K 2024-12-04T09:45:10,463 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7f96cfe0d506497391c6c6c9a7d10e6e as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7f96cfe0d506497391c6c6c9a7d10e6e 2024-12-04T09:45:10,468 INFO [M:0;84486a41f81c:38097 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7f96cfe0d506497391c6c6c9a7d10e6e 2024-12-04T09:45:10,468 INFO [M:0;84486a41f81c:38097 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7f96cfe0d506497391c6c6c9a7d10e6e, entries=17, sequenceid=168, filesize=7.6 K 2024-12-04T09:45:10,470 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d5d9cef2cfd243329bade5c07bdb2575 as hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d5d9cef2cfd243329bade5c07bdb2575 2024-12-04T09:45:10,475 INFO [M:0;84486a41f81c:38097 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d5d9cef2cfd243329bade5c07bdb2575 2024-12-04T09:45:10,476 INFO [M:0;84486a41f81c:38097 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33391/user/jenkins/test-data/cacccfd4-87c9-bc67-bc32-b2990b855095/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d5d9cef2cfd243329bade5c07bdb2575, entries=3, sequenceid=168, filesize=5.3 K 2024-12-04T09:45:10,477 INFO [M:0;84486a41f81c:38097 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=168, compaction requested=false 2024-12-04T09:45:10,478 INFO [M:0;84486a41f81c:38097 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T09:45:10,478 DEBUG [M:0;84486a41f81c:38097 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733305510372Disabling compacts and flushes for region at 1733305510372Disabling writes for close at 1733305510372Obtaining lock to block concurrent updates at 1733305510372Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733305510372Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1733305510372Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733305510373 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733305510373Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733305510385 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733305510385Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733305510397 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733305510411 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733305510411Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733305510424 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733305510438 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733305510438Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e3d8479: reopening flushed file at 1733305510454 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6970c339: reopening flushed file at 1733305510462 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7067c8fa: reopening flushed file at 1733305510469 (+7 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 105ms, sequenceid=168, compaction requested=false at 1733305510477 (+8 ms)Writing region close event to WAL at 1733305510478 (+1 ms)Closed at 1733305510478 2024-12-04T09:45:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40851 is added to blk_1073741830_1006 (size=71337) 2024-12-04T09:45:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36275 is added to blk_1073741830_1006 (size=71337) 2024-12-04T09:45:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33979 is added to blk_1073741830_1006 (size=71337) 2024-12-04T09:45:10,481 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T09:45:10,482 INFO [M:0;84486a41f81c:38097 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T09:45:10,482 INFO [M:0;84486a41f81c:38097 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38097 2024-12-04T09:45:10,482 INFO [M:0;84486a41f81c:38097 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T09:45:10,592 INFO [M:0;84486a41f81c:38097 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T09:45:10,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:45:10,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38097-0x101a10432830000, quorum=127.0.0.1:64397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T09:45:10,599 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305490208 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305490208 (inode 16655) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733305489901/wal.1733305490208 (inode 16655) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-04T09:45:10,599 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733305481954/wal.1733305482055 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:10,599 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733305505354/wal.1733305505633 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:10,601 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305499606 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305499606 (inode 16678) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733305490341/wal.1733305499606 (inode 16678) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-04T09:45:10,602 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733305475560/wal.1733305475624 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:10,604 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305500348 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305500348 (inode 16704) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733305499832/wal.1733305500348 (inode 16704) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-04T09:45:10,605 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733305500494/wal.1733305500590 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T09:45:10,607 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal.1733305475434 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal.1733305475434 (inode 16485) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733305475267/wal.1733305475434 (inode 16485) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-04T09:45:10,609 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal.1733305475034 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal.1733305475034 (inode 16462) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733305474814/wal.1733305475034 (inode 16462) Holder DFSClient_NONMAPREDUCE_-2050232587_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-04T09:45:10,613 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@31a39bd8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T09:45:10,616 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5aa63b11{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T09:45:10,616 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T09:45:10,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de3b13{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T09:45:10,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d458b2a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,STOPPED} 2024-12-04T09:45:10,619 WARN [BP-129416318-172.17.0.2-1733305468255 heartbeating to localhost/127.0.0.1:33391 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T09:45:10,619 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T09:45:10,619 WARN [BP-129416318-172.17.0.2-1733305468255 heartbeating to localhost/127.0.0.1:33391 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-129416318-172.17.0.2-1733305468255 (Datanode Uuid 784e3a01-d4c8-46ff-b691-fd45daadc952) service to localhost/127.0.0.1:33391 2024-12-04T09:45:10,619 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T09:45:10,620 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data5/current/BP-129416318-172.17.0.2-1733305468255 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T09:45:10,621 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data6/current/BP-129416318-172.17.0.2-1733305468255 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T09:45:10,621 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T09:45:10,623 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@41fe4ca9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T09:45:10,624 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@74ee88d2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T09:45:10,624 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T09:45:10,624 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@424d969a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T09:45:10,624 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@157ee20f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,STOPPED} 2024-12-04T09:45:10,625 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T09:45:10,625 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T09:45:10,625 WARN [BP-129416318-172.17.0.2-1733305468255 heartbeating to localhost/127.0.0.1:33391 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T09:45:10,625 WARN [BP-129416318-172.17.0.2-1733305468255 heartbeating to localhost/127.0.0.1:33391 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-129416318-172.17.0.2-1733305468255 (Datanode Uuid d8d59ee4-b57e-4493-9cb4-2891c68d537e) service to localhost/127.0.0.1:33391 2024-12-04T09:45:10,626 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data3/current/BP-129416318-172.17.0.2-1733305468255 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T09:45:10,626 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data4/current/BP-129416318-172.17.0.2-1733305468255 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T09:45:10,626 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T09:45:10,628 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3a1c1498{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T09:45:10,629 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3badc187{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T09:45:10,629 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T09:45:10,629 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@991988e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T09:45:10,629 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@481eb1b6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,STOPPED} 2024-12-04T09:45:10,630 WARN [BP-129416318-172.17.0.2-1733305468255 heartbeating to localhost/127.0.0.1:33391 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T09:45:10,630 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T09:45:10,630 WARN [BP-129416318-172.17.0.2-1733305468255 heartbeating to localhost/127.0.0.1:33391 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-129416318-172.17.0.2-1733305468255 (Datanode Uuid 5db4c2f1-0af9-47e1-9734-52e71841f250) service to localhost/127.0.0.1:33391 2024-12-04T09:45:10,630 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T09:45:10,631 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data1/current/BP-129416318-172.17.0.2-1733305468255 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T09:45:10,631 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/cluster_1dae7826-142d-cc22-d0ce-a37bc404a01b/data/data2/current/BP-129416318-172.17.0.2-1733305468255 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T09:45:10,631 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T09:45:10,637 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@12264dc8{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T09:45:10,638 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ce294cb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T09:45:10,638 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T09:45:10,638 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43ac9415{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T09:45:10,638 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@77da9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/b53a349d-9d6a-bf4d-bec3-28d1f4f52b56/hadoop.log.dir/,STOPPED} 2024-12-04T09:45:10,646 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T09:45:10,689 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down