2024-12-03 14:51:37,554 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 14:51:37,569 main DEBUG Took 0.013022 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 14:51:37,570 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 14:51:37,570 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 14:51:37,571 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 14:51:37,573 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,580 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 14:51:37,595 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,598 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,599 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,601 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,602 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,603 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,604 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,604 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,605 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,606 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,607 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,607 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,608 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,608 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,609 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 14:51:37,610 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,610 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 14:51:37,612 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 14:51:37,614 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 14:51:37,616 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 14:51:37,616 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 14:51:37,618 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 14:51:37,618 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 14:51:37,628 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 14:51:37,631 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 14:51:37,632 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 14:51:37,633 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 14:51:37,633 main DEBUG createAppenders(={Console}) 2024-12-03 14:51:37,634 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-03 14:51:37,634 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 14:51:37,635 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-03 14:51:37,635 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 14:51:37,636 main DEBUG OutputStream closed 2024-12-03 14:51:37,636 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 14:51:37,636 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 14:51:37,637 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-03 14:51:37,701 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 14:51:37,703 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 14:51:37,704 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 14:51:37,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 14:51:37,705 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 14:51:37,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 14:51:37,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 14:51:37,706 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 14:51:37,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 14:51:37,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 14:51:37,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 14:51:37,707 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 14:51:37,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 14:51:37,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 14:51:37,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 14:51:37,708 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 14:51:37,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 14:51:37,709 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 14:51:37,711 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 14:51:37,712 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-03 14:51:37,712 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 14:51:37,713 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-03T14:51:37,953 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e 2024-12-03 14:51:37,956 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 14:51:37,957 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T14:51:37,965 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-03T14:51:37,971 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-12-03T14:51:37,990 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T14:51:38,030 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T14:51:38,030 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T14:51:38,041 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T14:51:38,054 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0, deleteOnExit=true 2024-12-03T14:51:38,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T14:51:38,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/test.cache.data in system properties and HBase conf 2024-12-03T14:51:38,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T14:51:38,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir in system properties and HBase conf 2024-12-03T14:51:38,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T14:51:38,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T14:51:38,057 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T14:51:38,128 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T14:51:38,206 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T14:51:38,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T14:51:38,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T14:51:38,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T14:51:38,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T14:51:38,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T14:51:38,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T14:51:38,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T14:51:38,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T14:51:38,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T14:51:38,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/nfs.dump.dir in system properties and HBase conf 2024-12-03T14:51:38,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/java.io.tmpdir in system properties and HBase conf 2024-12-03T14:51:38,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T14:51:38,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T14:51:38,214 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T14:51:38,866 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T14:51:38,938 INFO [Time-limited test {}] log.Log(170): Logging initialized @2049ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T14:51:39,005 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:51:39,058 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:51:39,076 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:51:39,076 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:51:39,077 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T14:51:39,089 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:51:39,092 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:51:39,093 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:51:39,263 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c8b857b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/java.io.tmpdir/jetty-localhost-38487-hadoop-hdfs-3_4_1-tests_jar-_-any-9618529660799220964/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T14:51:39,268 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:38487} 2024-12-03T14:51:39,269 INFO [Time-limited test {}] server.Server(415): Started @2380ms 2024-12-03T14:51:39,580 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:51:39,586 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:51:39,587 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:51:39,587 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:51:39,587 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T14:51:39,588 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:51:39,588 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:51:39,682 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17d1a8d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/java.io.tmpdir/jetty-localhost-44831-hadoop-hdfs-3_4_1-tests_jar-_-any-15038157104475774122/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:51:39,683 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:44831} 2024-12-03T14:51:39,683 INFO [Time-limited test {}] server.Server(415): Started @2794ms 2024-12-03T14:51:39,727 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T14:51:39,843 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:51:39,850 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:51:39,851 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:51:39,852 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:51:39,852 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T14:51:39,854 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:51:39,855 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:51:39,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4464725f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/java.io.tmpdir/jetty-localhost-41267-hadoop-hdfs-3_4_1-tests_jar-_-any-16823685814339817437/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:51:39,985 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:41267} 2024-12-03T14:51:39,985 INFO [Time-limited test {}] server.Server(415): Started @3097ms 2024-12-03T14:51:39,987 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T14:51:40,026 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T14:51:40,033 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T14:51:40,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T14:51:40,038 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T14:51:40,038 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T14:51:40,039 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,AVAILABLE} 2024-12-03T14:51:40,040 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T14:51:40,100 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data1/current/BP-870883219-172.17.0.2-1733237498670/current, will proceed with Du for space computation calculation, 2024-12-03T14:51:40,100 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data3/current/BP-870883219-172.17.0.2-1733237498670/current, will proceed with Du for space computation calculation, 2024-12-03T14:51:40,100 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data4/current/BP-870883219-172.17.0.2-1733237498670/current, will proceed with Du for space computation calculation, 2024-12-03T14:51:40,100 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data2/current/BP-870883219-172.17.0.2-1733237498670/current, will proceed with Du for space computation calculation, 2024-12-03T14:51:40,146 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T14:51:40,146 WARN [Thread-83 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T14:51:40,156 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26f8cb79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/java.io.tmpdir/jetty-localhost-35927-hadoop-hdfs-3_4_1-tests_jar-_-any-18133088593212182116/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:51:40,157 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:35927} 2024-12-03T14:51:40,158 INFO [Time-limited test {}] server.Server(415): Started @3269ms 2024-12-03T14:51:40,160 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T14:51:40,209 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9a029837533f7b03 with lease ID 0x378ad66f4c2f1554: Processing first storage report for DS-110a8131-74a0-42b4-9d36-f29cd08198b9 from datanode DatanodeRegistration(127.0.0.1:40191, datanodeUuid=85a55e3f-948b-4b5c-a683-9fce9149759b, infoPort=44869, infoSecurePort=0, ipcPort=46259, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670) 2024-12-03T14:51:40,210 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a029837533f7b03 with lease ID 0x378ad66f4c2f1554: from storage DS-110a8131-74a0-42b4-9d36-f29cd08198b9 node DatanodeRegistration(127.0.0.1:40191, datanodeUuid=85a55e3f-948b-4b5c-a683-9fce9149759b, infoPort=44869, infoSecurePort=0, ipcPort=46259, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T14:51:40,211 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeda612332cca5c7d with lease ID 0x378ad66f4c2f1553: Processing first storage report for DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505 from datanode DatanodeRegistration(127.0.0.1:38931, datanodeUuid=c42b874a-68d0-4d7c-b8c3-72f99028c692, infoPort=36271, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670) 2024-12-03T14:51:40,211 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeda612332cca5c7d with lease ID 0x378ad66f4c2f1553: from storage DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505 node DatanodeRegistration(127.0.0.1:38931, datanodeUuid=c42b874a-68d0-4d7c-b8c3-72f99028c692, infoPort=36271, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T14:51:40,212 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9a029837533f7b03 with lease ID 0x378ad66f4c2f1554: Processing first storage report for DS-0890b658-6bc1-425d-8626-f5c5786fbc35 from datanode DatanodeRegistration(127.0.0.1:40191, datanodeUuid=85a55e3f-948b-4b5c-a683-9fce9149759b, infoPort=44869, infoSecurePort=0, ipcPort=46259, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670) 2024-12-03T14:51:40,212 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a029837533f7b03 with lease ID 0x378ad66f4c2f1554: from storage DS-0890b658-6bc1-425d-8626-f5c5786fbc35 node DatanodeRegistration(127.0.0.1:40191, datanodeUuid=85a55e3f-948b-4b5c-a683-9fce9149759b, infoPort=44869, infoSecurePort=0, ipcPort=46259, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T14:51:40,212 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xeda612332cca5c7d with lease ID 0x378ad66f4c2f1553: Processing first storage report for DS-4be881e0-8cda-4af3-958c-44f845496de6 from datanode DatanodeRegistration(127.0.0.1:38931, datanodeUuid=c42b874a-68d0-4d7c-b8c3-72f99028c692, infoPort=36271, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670) 2024-12-03T14:51:40,213 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xeda612332cca5c7d with lease ID 0x378ad66f4c2f1553: from storage DS-4be881e0-8cda-4af3-958c-44f845496de6 node DatanodeRegistration(127.0.0.1:38931, datanodeUuid=c42b874a-68d0-4d7c-b8c3-72f99028c692, infoPort=36271, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T14:51:40,257 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data5/current/BP-870883219-172.17.0.2-1733237498670/current, will proceed with Du for space computation calculation, 2024-12-03T14:51:40,257 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data6/current/BP-870883219-172.17.0.2-1733237498670/current, will proceed with Du for space computation calculation, 2024-12-03T14:51:40,280 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T14:51:40,284 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3235f7321c04417f with lease ID 0x378ad66f4c2f1555: Processing first storage report for DS-64dc720f-7496-41d5-bed7-d1f64d66c987 from datanode DatanodeRegistration(127.0.0.1:37353, datanodeUuid=b785c90a-24ce-4fac-ac2d-3b2d6c761cec, infoPort=36369, infoSecurePort=0, ipcPort=33221, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670) 2024-12-03T14:51:40,285 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3235f7321c04417f with lease ID 0x378ad66f4c2f1555: from storage DS-64dc720f-7496-41d5-bed7-d1f64d66c987 node DatanodeRegistration(127.0.0.1:37353, datanodeUuid=b785c90a-24ce-4fac-ac2d-3b2d6c761cec, infoPort=36369, infoSecurePort=0, ipcPort=33221, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T14:51:40,285 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3235f7321c04417f with lease ID 0x378ad66f4c2f1555: Processing first storage report for DS-2c73a1e4-ae4c-43d4-90aa-ca6aec32dbf0 from datanode DatanodeRegistration(127.0.0.1:37353, datanodeUuid=b785c90a-24ce-4fac-ac2d-3b2d6c761cec, infoPort=36369, infoSecurePort=0, ipcPort=33221, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670) 2024-12-03T14:51:40,285 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3235f7321c04417f with lease ID 0x378ad66f4c2f1555: from storage DS-2c73a1e4-ae4c-43d4-90aa-ca6aec32dbf0 node DatanodeRegistration(127.0.0.1:37353, datanodeUuid=b785c90a-24ce-4fac-ac2d-3b2d6c761cec, infoPort=36369, infoSecurePort=0, ipcPort=33221, storageInfo=lv=-57;cid=testClusterID;nsid=1511565275;c=1733237498670), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T14:51:40,518 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e 2024-12-03T14:51:40,613 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/zookeeper_0, clientPort=51554, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T14:51:40,624 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51554 2024-12-03T14:51:40,633 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:40,636 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:40,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741825_1001 (size=7) 2024-12-03T14:51:40,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741825_1001 (size=7) 2024-12-03T14:51:40,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741825_1001 (size=7) 2024-12-03T14:51:41,244 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918 with version=8 2024-12-03T14:51:41,244 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/hbase-staging 2024-12-03T14:51:41,481 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a5d22df9eca2:0 server-side Connection retries=45 2024-12-03T14:51:41,489 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,494 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:51:41,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,494 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:51:41,610 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T14:51:41,664 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T14:51:41,672 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T14:51:41,675 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:51:41,702 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 103307 (auto-detected) 2024-12-03T14:51:41,704 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T14:51:41,723 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46647 2024-12-03T14:51:41,742 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46647 connecting to ZooKeeper ensemble=127.0.0.1:51554 2024-12-03T14:51:41,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:466470x0, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:51:41,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46647-0x100a090c71d0000 connected 2024-12-03T14:51:41,795 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:41,799 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:41,809 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:51:41,813 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918, hbase.cluster.distributed=false 2024-12-03T14:51:41,835 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:51:41,839 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46647 2024-12-03T14:51:41,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46647 2024-12-03T14:51:41,840 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46647 2024-12-03T14:51:41,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46647 2024-12-03T14:51:41,841 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46647 2024-12-03T14:51:41,932 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45 2024-12-03T14:51:41,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,934 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:51:41,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:51:41,937 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T14:51:41,939 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:51:41,940 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43019 2024-12-03T14:51:41,942 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43019 connecting to ZooKeeper ensemble=127.0.0.1:51554 2024-12-03T14:51:41,943 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:41,946 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:41,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:430190x0, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:51:41,952 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:51:41,952 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43019-0x100a090c71d0001 connected 2024-12-03T14:51:41,956 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T14:51:41,964 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T14:51:41,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T14:51:41,971 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:51:41,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43019 2024-12-03T14:51:41,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43019 2024-12-03T14:51:41,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43019 2024-12-03T14:51:41,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43019 2024-12-03T14:51:41,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43019 2024-12-03T14:51:41,989 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45 2024-12-03T14:51:41,989 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,990 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:51:41,990 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:41,991 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:51:41,991 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T14:51:41,991 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:51:41,992 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43671 2024-12-03T14:51:41,994 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43671 connecting to ZooKeeper ensemble=127.0.0.1:51554 2024-12-03T14:51:41,995 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:41,998 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:42,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436710x0, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:51:42,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436710x0, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:51:42,003 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43671-0x100a090c71d0002 connected 2024-12-03T14:51:42,003 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T14:51:42,004 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T14:51:42,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T14:51:42,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:51:42,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43671 2024-12-03T14:51:42,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43671 2024-12-03T14:51:42,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43671 2024-12-03T14:51:42,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43671 2024-12-03T14:51:42,011 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43671 2024-12-03T14:51:42,025 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a5d22df9eca2:0 server-side Connection retries=45 2024-12-03T14:51:42,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:42,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:42,026 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T14:51:42,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T14:51:42,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T14:51:42,026 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T14:51:42,027 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T14:51:42,028 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38259 2024-12-03T14:51:42,030 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38259 connecting to ZooKeeper ensemble=127.0.0.1:51554 2024-12-03T14:51:42,031 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:42,034 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:42,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382590x0, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T14:51:42,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38259-0x100a090c71d0003 connected 2024-12-03T14:51:42,041 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:382590x0, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:51:42,041 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T14:51:42,042 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T14:51:42,043 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T14:51:42,045 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T14:51:42,046 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38259 2024-12-03T14:51:42,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38259 2024-12-03T14:51:42,049 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38259 2024-12-03T14:51:42,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38259 2024-12-03T14:51:42,050 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38259 2024-12-03T14:51:42,064 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a5d22df9eca2:46647 2024-12-03T14:51:42,065 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:42,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,074 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:42,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T14:51:42,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T14:51:42,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T14:51:42,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,100 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T14:51:42,101 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a5d22df9eca2,46647,1733237501335 from backup master directory 2024-12-03T14:51:42,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:42,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T14:51:42,104 WARN [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:51:42,104 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:42,106 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T14:51:42,108 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T14:51:42,159 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/hbase.id] with ID: 1ef03b02-7d7a-4dc2-9ea3-fcb035dd7514 2024-12-03T14:51:42,159 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/.tmp/hbase.id 2024-12-03T14:51:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741826_1002 (size=42) 2024-12-03T14:51:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741826_1002 (size=42) 2024-12-03T14:51:42,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741826_1002 (size=42) 2024-12-03T14:51:42,174 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/.tmp/hbase.id]:[hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/hbase.id] 2024-12-03T14:51:42,221 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T14:51:42,225 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T14:51:42,244 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-12-03T14:51:42,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741827_1003 (size=196) 2024-12-03T14:51:42,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741827_1003 (size=196) 2024-12-03T14:51:42,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741827_1003 (size=196) 2024-12-03T14:51:42,280 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:51:42,281 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T14:51:42,286 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:42,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741828_1004 (size=1189) 2024-12-03T14:51:42,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741828_1004 (size=1189) 2024-12-03T14:51:42,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741828_1004 (size=1189) 2024-12-03T14:51:42,330 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store 2024-12-03T14:51:42,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741829_1005 (size=34) 2024-12-03T14:51:42,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741829_1005 (size=34) 2024-12-03T14:51:42,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741829_1005 (size=34) 2024-12-03T14:51:42,358 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T14:51:42,361 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:42,362 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T14:51:42,362 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:51:42,362 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:51:42,364 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T14:51:42,364 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:51:42,364 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:51:42,365 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733237502362Disabling compacts and flushes for region at 1733237502362Disabling writes for close at 1733237502364 (+2 ms)Writing region close event to WAL at 1733237502364Closed at 1733237502364 2024-12-03T14:51:42,367 WARN [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/.initializing 2024-12-03T14:51:42,367 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/WALs/a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:42,374 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:42,387 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C46647%2C1733237501335, suffix=, logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/WALs/a5d22df9eca2,46647,1733237501335, archiveDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/oldWALs, maxLogs=10 2024-12-03T14:51:42,426 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/WALs/a5d22df9eca2,46647,1733237501335/a5d22df9eca2%2C46647%2C1733237501335.1733237502393, exclude list is [], retry=0 2024-12-03T14:51:42,443 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:42,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:42,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:42,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:42,451 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T14:51:42,485 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/WALs/a5d22df9eca2,46647,1733237501335/a5d22df9eca2%2C46647%2C1733237501335.1733237502393 2024-12-03T14:51:42,486 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:42,486 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:42,487 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:42,490 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,491 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T14:51:42,549 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:42,552 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:42,552 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,556 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T14:51:42,556 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:42,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:42,557 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T14:51:42,560 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:42,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:42,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,564 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T14:51:42,564 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:42,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:42,566 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,569 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,570 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,576 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,576 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,580 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T14:51:42,583 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T14:51:42,587 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:42,588 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59975849, jitterRate=-0.10629020631313324}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T14:51:42,594 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733237502503Initializing all the Stores at 1733237502505 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237502505Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237502506 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237502506Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237502507 (+1 ms)Cleaning up temporary data from old regions at 1733237502577 (+70 ms)Region opened successfully at 1733237502594 (+17 ms) 2024-12-03T14:51:42,595 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T14:51:42,628 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e4adda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0 2024-12-03T14:51:42,656 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T14:51:42,666 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T14:51:42,666 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T14:51:42,668 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T14:51:42,669 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T14:51:42,673 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T14:51:42,673 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T14:51:42,696 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T14:51:42,704 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T14:51:42,706 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T14:51:42,708 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T14:51:42,709 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T14:51:42,710 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T14:51:42,712 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T14:51:42,715 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T14:51:42,716 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T14:51:42,718 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T14:51:42,719 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T14:51:42,734 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T14:51:42,735 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T14:51:42,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:51:42,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:51:42,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:51:42,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T14:51:42,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,742 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a5d22df9eca2,46647,1733237501335, sessionid=0x100a090c71d0000, setting cluster-up flag (Was=false) 2024-12-03T14:51:42,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,758 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T14:51:42,759 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:42,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:42,768 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T14:51:42,770 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:42,776 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T14:51:42,844 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T14:51:42,852 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T14:51:42,854 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(746): ClusterId : 1ef03b02-7d7a-4dc2-9ea3-fcb035dd7514 2024-12-03T14:51:42,854 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(746): ClusterId : 1ef03b02-7d7a-4dc2-9ea3-fcb035dd7514 2024-12-03T14:51:42,854 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(746): ClusterId : 1ef03b02-7d7a-4dc2-9ea3-fcb035dd7514 2024-12-03T14:51:42,857 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T14:51:42,857 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T14:51:42,857 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T14:51:42,860 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T14:51:42,862 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T14:51:42,862 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T14:51:42,862 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T14:51:42,862 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T14:51:42,862 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T14:51:42,862 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T14:51:42,865 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T14:51:42,865 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T14:51:42,865 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T14:51:42,866 DEBUG [RS:2;a5d22df9eca2:38259 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a42b382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0 2024-12-03T14:51:42,866 DEBUG [RS:0;a5d22df9eca2:43019 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9110e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0 2024-12-03T14:51:42,866 DEBUG [RS:1;a5d22df9eca2:43671 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e35f81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a5d22df9eca2/172.17.0.2:0 2024-12-03T14:51:42,866 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a5d22df9eca2,46647,1733237501335 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T14:51:42,876 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:51:42,876 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:51:42,876 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:51:42,876 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=5, maxPoolSize=5 2024-12-03T14:51:42,877 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a5d22df9eca2:0, corePoolSize=10, maxPoolSize=10 2024-12-03T14:51:42,877 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:42,877 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:51:42,877 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:42,886 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;a5d22df9eca2:43671 2024-12-03T14:51:42,888 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a5d22df9eca2:43019 2024-12-03T14:51:42,890 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T14:51:42,890 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T14:51:42,891 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T14:51:42,891 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T14:51:42,891 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:51:42,891 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T14:51:42,892 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T14:51:42,893 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T14:51:42,893 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;a5d22df9eca2:38259 2024-12-03T14:51:42,893 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T14:51:42,893 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T14:51:42,893 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T14:51:42,894 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,46647,1733237501335 with port=43671, startcode=1733237501988 2024-12-03T14:51:42,894 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,46647,1733237501335 with port=43019, startcode=1733237501901 2024-12-03T14:51:42,895 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(2659): reportForDuty to master=a5d22df9eca2,46647,1733237501335 with port=38259, startcode=1733237502024 2024-12-03T14:51:42,900 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:42,900 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T14:51:42,901 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733237532901 2024-12-03T14:51:42,904 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T14:51:42,905 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T14:51:42,908 DEBUG [RS:0;a5d22df9eca2:43019 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T14:51:42,908 DEBUG [RS:2;a5d22df9eca2:38259 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T14:51:42,909 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T14:51:42,909 DEBUG [RS:1;a5d22df9eca2:43671 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T14:51:42,910 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T14:51:42,910 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T14:51:42,910 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T14:51:42,911 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:42,919 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T14:51:42,921 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T14:51:42,921 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T14:51:42,931 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T14:51:42,931 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T14:51:42,950 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237502949,5,FailOnTimeoutGroup] 2024-12-03T14:51:42,951 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237502950,5,FailOnTimeoutGroup] 2024-12-03T14:51:42,951 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:42,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741831_1007 (size=1321) 2024-12-03T14:51:42,953 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T14:51:42,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741831_1007 (size=1321) 2024-12-03T14:51:42,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741831_1007 (size=1321) 2024-12-03T14:51:42,955 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:42,955 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:42,955 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T14:51:42,956 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918 2024-12-03T14:51:42,970 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36395, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T14:51:42,970 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44383, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T14:51:42,970 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58545, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T14:51:42,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741832_1008 (size=32) 2024-12-03T14:51:42,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741832_1008 (size=32) 2024-12-03T14:51:42,978 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:42,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741832_1008 (size=32) 2024-12-03T14:51:42,980 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:42,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46647 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:42,983 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T14:51:42,986 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T14:51:42,986 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:42,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:42,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T14:51:42,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T14:51:42,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:42,992 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,38259,1733237502024 2024-12-03T14:51:42,992 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46647 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,38259,1733237502024 2024-12-03T14:51:42,992 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:42,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T14:51:42,996 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918 2024-12-03T14:51:42,996 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38155 2024-12-03T14:51:42,996 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T14:51:42,998 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46647 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:42,998 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46647 {}] master.ServerManager(517): Registering regionserver=a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:42,998 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918 2024-12-03T14:51:42,998 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38155 2024-12-03T14:51:42,998 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T14:51:42,999 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T14:51:42,999 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:43,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:43,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:51:43,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T14:51:43,003 DEBUG [RS:2;a5d22df9eca2:38259 {}] zookeeper.ZKUtil(111): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,38259,1733237502024 2024-12-03T14:51:43,003 WARN [RS:2;a5d22df9eca2:38259 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:51:43,004 INFO [RS:2;a5d22df9eca2:38259 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:43,004 DEBUG [RS:1;a5d22df9eca2:43671 {}] zookeeper.ZKUtil(111): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:43,004 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,38259,1733237502024 2024-12-03T14:51:43,004 WARN [RS:1;a5d22df9eca2:43671 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:51:43,004 INFO [RS:1;a5d22df9eca2:43671 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:43,004 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918 2024-12-03T14:51:43,004 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:43,005 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38155 2024-12-03T14:51:43,005 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T14:51:43,005 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,43671,1733237501988] 2024-12-03T14:51:43,006 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,38259,1733237502024] 2024-12-03T14:51:43,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T14:51:43,007 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:43,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:51:43,008 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a5d22df9eca2,43019,1733237501901] 2024-12-03T14:51:43,008 DEBUG [RS:0;a5d22df9eca2:43019 {}] zookeeper.ZKUtil(111): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:43,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:43,009 WARN [RS:0;a5d22df9eca2:43019 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T14:51:43,009 INFO [RS:0;a5d22df9eca2:43019 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:43,009 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T14:51:43,009 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:43,010 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740 2024-12-03T14:51:43,013 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740 2024-12-03T14:51:43,017 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T14:51:43,017 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T14:51:43,018 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T14:51:43,022 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T14:51:43,027 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:43,028 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71071963, jitterRate=0.05905477702617645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T14:51:43,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733237502980Initializing all the Stores at 1733237502982 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237502983 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237502983Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237502983Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237502983Cleaning up temporary data from old regions at 1733237503017 (+34 ms)Region opened successfully at 1733237503030 (+13 ms) 2024-12-03T14:51:43,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T14:51:43,030 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T14:51:43,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T14:51:43,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T14:51:43,030 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T14:51:43,031 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T14:51:43,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733237503030Disabling compacts and flushes for region at 1733237503030Disabling writes for close at 1733237503030Writing region close event to WAL at 1733237503031 (+1 ms)Closed at 1733237503031 2024-12-03T14:51:43,034 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:51:43,034 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T14:51:43,036 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T14:51:43,036 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T14:51:43,036 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T14:51:43,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T14:51:43,049 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T14:51:43,049 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T14:51:43,049 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T14:51:43,050 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T14:51:43,054 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T14:51:43,055 INFO [RS:0;a5d22df9eca2:43019 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T14:51:43,055 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,056 INFO [RS:2;a5d22df9eca2:38259 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T14:51:43,056 INFO [RS:1;a5d22df9eca2:43671 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T14:51:43,056 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,056 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,058 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T14:51:43,058 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T14:51:43,058 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T14:51:43,063 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T14:51:43,063 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T14:51:43,063 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T14:51:43,065 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,065 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,065 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,065 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:51:43,065 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,065 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:51:43,066 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0, corePoolSize=2, maxPoolSize=2 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:51:43,066 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:1;a5d22df9eca2:43671 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:51:43,066 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,066 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a5d22df9eca2:0, corePoolSize=1, maxPoolSize=1 2024-12-03T14:51:43,067 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:51:43,067 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:51:43,067 DEBUG [RS:0;a5d22df9eca2:43019 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:51:43,067 DEBUG [RS:2;a5d22df9eca2:38259 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a5d22df9eca2:0, corePoolSize=3, maxPoolSize=3 2024-12-03T14:51:43,067 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,43019,1733237501901-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,43671,1733237501988-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,068 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,38259,1733237502024-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:51:43,084 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T14:51:43,084 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T14:51:43,084 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T14:51:43,085 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,43671,1733237501988-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,085 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,43019,1733237501901-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,085 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,38259,1733237502024-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,086 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,086 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,086 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,086 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.Replication(171): a5d22df9eca2,38259,1733237502024 started 2024-12-03T14:51:43,086 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.Replication(171): a5d22df9eca2,43019,1733237501901 started 2024-12-03T14:51:43,086 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.Replication(171): a5d22df9eca2,43671,1733237501988 started 2024-12-03T14:51:43,100 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,100 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,100 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,101 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,43019,1733237501901, RpcServer on a5d22df9eca2/172.17.0.2:43019, sessionid=0x100a090c71d0001 2024-12-03T14:51:43,101 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,38259,1733237502024, RpcServer on a5d22df9eca2/172.17.0.2:38259, sessionid=0x100a090c71d0003 2024-12-03T14:51:43,101 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1482): Serving as a5d22df9eca2,43671,1733237501988, RpcServer on a5d22df9eca2/172.17.0.2:43671, sessionid=0x100a090c71d0002 2024-12-03T14:51:43,101 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T14:51:43,101 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T14:51:43,101 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T14:51:43,101 DEBUG [RS:2;a5d22df9eca2:38259 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,38259,1733237502024 2024-12-03T14:51:43,101 DEBUG [RS:1;a5d22df9eca2:43671 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:43,101 DEBUG [RS:0;a5d22df9eca2:43019 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:43,102 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,38259,1733237502024' 2024-12-03T14:51:43,102 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,43019,1733237501901' 2024-12-03T14:51:43,102 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,43671,1733237501988' 2024-12-03T14:51:43,102 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T14:51:43,102 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T14:51:43,102 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T14:51:43,102 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T14:51:43,102 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T14:51:43,103 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T14:51:43,103 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T14:51:43,103 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T14:51:43,103 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T14:51:43,103 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T14:51:43,103 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T14:51:43,103 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T14:51:43,103 DEBUG [RS:0;a5d22df9eca2:43019 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:43,103 DEBUG [RS:2;a5d22df9eca2:38259 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,38259,1733237502024 2024-12-03T14:51:43,103 DEBUG [RS:1;a5d22df9eca2:43671 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:43,103 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,43019,1733237501901' 2024-12-03T14:51:43,103 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,43671,1733237501988' 2024-12-03T14:51:43,103 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a5d22df9eca2,38259,1733237502024' 2024-12-03T14:51:43,103 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T14:51:43,103 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T14:51:43,103 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T14:51:43,104 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T14:51:43,104 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T14:51:43,104 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T14:51:43,104 DEBUG [RS:0;a5d22df9eca2:43019 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T14:51:43,104 INFO [RS:0;a5d22df9eca2:43019 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T14:51:43,104 DEBUG [RS:1;a5d22df9eca2:43671 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T14:51:43,105 INFO [RS:0;a5d22df9eca2:43019 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T14:51:43,105 INFO [RS:1;a5d22df9eca2:43671 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T14:51:43,105 INFO [RS:1;a5d22df9eca2:43671 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T14:51:43,105 DEBUG [RS:2;a5d22df9eca2:38259 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T14:51:43,105 INFO [RS:2;a5d22df9eca2:38259 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T14:51:43,105 INFO [RS:2;a5d22df9eca2:38259 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T14:51:43,206 WARN [a5d22df9eca2:46647 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T14:51:43,217 INFO [RS:2;a5d22df9eca2:38259 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:43,217 INFO [RS:0;a5d22df9eca2:43019 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:43,217 INFO [RS:1;a5d22df9eca2:43671 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:43,220 INFO [RS:2;a5d22df9eca2:38259 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C38259%2C1733237502024, suffix=, logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,38259,1733237502024, archiveDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs, maxLogs=32 2024-12-03T14:51:43,220 INFO [RS:1;a5d22df9eca2:43671 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C43671%2C1733237501988, suffix=, logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988, archiveDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs, maxLogs=32 2024-12-03T14:51:43,220 INFO [RS:0;a5d22df9eca2:43019 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C43019%2C1733237501901, suffix=, logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901, archiveDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs, maxLogs=32 2024-12-03T14:51:43,236 DEBUG [RS:2;a5d22df9eca2:38259 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,38259,1733237502024/a5d22df9eca2%2C38259%2C1733237502024.1733237503225, exclude list is [], retry=0 2024-12-03T14:51:43,236 DEBUG [RS:1;a5d22df9eca2:43671 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988/a5d22df9eca2%2C43671%2C1733237501988.1733237503225, exclude list is [], retry=0 2024-12-03T14:51:43,237 DEBUG [RS:0;a5d22df9eca2:43019 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901/a5d22df9eca2%2C43019%2C1733237501901.1733237503225, exclude list is [], retry=0 2024-12-03T14:51:43,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:43,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:43,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:43,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:43,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:43,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:43,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:43,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:43,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:43,249 INFO [RS:2;a5d22df9eca2:38259 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,38259,1733237502024/a5d22df9eca2%2C38259%2C1733237502024.1733237503225 2024-12-03T14:51:43,250 DEBUG [RS:2;a5d22df9eca2:38259 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:43,251 INFO [RS:0;a5d22df9eca2:43019 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 2024-12-03T14:51:43,252 DEBUG [RS:0;a5d22df9eca2:43019 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:43,253 INFO [RS:1;a5d22df9eca2:43671 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988/a5d22df9eca2%2C43671%2C1733237501988.1733237503225 2024-12-03T14:51:43,254 DEBUG [RS:1;a5d22df9eca2:43671 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:43,462 DEBUG [a5d22df9eca2:46647 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T14:51:43,471 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(204): Hosts are {a5d22df9eca2=0} racks are {/default-rack=0} 2024-12-03T14:51:43,479 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T14:51:43,479 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T14:51:43,479 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T14:51:43,479 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T14:51:43,479 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T14:51:43,479 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T14:51:43,479 INFO [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T14:51:43,479 INFO [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T14:51:43,479 INFO [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T14:51:43,479 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T14:51:43,486 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:43,491 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,43671,1733237501988, state=OPENING 2024-12-03T14:51:43,495 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T14:51:43,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:43,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:43,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:43,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:51:43,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,499 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T14:51:43,501 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,43671,1733237501988}] 2024-12-03T14:51:43,678 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T14:51:43,680 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51409, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T14:51:43,692 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T14:51:43,692 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:43,693 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T14:51:43,696 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a5d22df9eca2%2C43671%2C1733237501988.meta, suffix=.meta, logDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988, archiveDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs, maxLogs=32 2024-12-03T14:51:43,710 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988/a5d22df9eca2%2C43671%2C1733237501988.meta.1733237503698.meta, exclude list is [], retry=0 2024-12-03T14:51:43,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:43,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:43,714 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:43,717 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988/a5d22df9eca2%2C43671%2C1733237501988.meta.1733237503698.meta 2024-12-03T14:51:43,717 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:43,717 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:43,719 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T14:51:43,721 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T14:51:43,726 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T14:51:43,730 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T14:51:43,731 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:43,731 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T14:51:43,731 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T14:51:43,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T14:51:43,735 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T14:51:43,735 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:43,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:43,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T14:51:43,738 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T14:51:43,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:43,738 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:43,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T14:51:43,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T14:51:43,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:43,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:43,741 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T14:51:43,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T14:51:43,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:43,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T14:51:43,743 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T14:51:43,744 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740 2024-12-03T14:51:43,747 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740 2024-12-03T14:51:43,749 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T14:51:43,749 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T14:51:43,750 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T14:51:43,753 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T14:51:43,754 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74109395, jitterRate=0.10431604087352753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T14:51:43,754 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T14:51:43,756 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733237503731Writing region info on filesystem at 1733237503732 (+1 ms)Initializing all the Stores at 1733237503733 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237503734 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237503734Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237503734Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733237503734Cleaning up temporary data from old regions at 1733237503749 (+15 ms)Running coprocessor post-open hooks at 1733237503754 (+5 ms)Region opened successfully at 1733237503756 (+2 ms) 2024-12-03T14:51:43,762 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733237503669 2024-12-03T14:51:43,773 DEBUG [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T14:51:43,774 INFO [RS_OPEN_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T14:51:43,775 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:43,778 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a5d22df9eca2,43671,1733237501988, state=OPEN 2024-12-03T14:51:43,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:51:43,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:51:43,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:51:43,779 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T14:51:43,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,780 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T14:51:43,780 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:43,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T14:51:43,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a5d22df9eca2,43671,1733237501988 in 281 msec 2024-12-03T14:51:43,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T14:51:43,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 750 msec 2024-12-03T14:51:43,797 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T14:51:43,797 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T14:51:43,821 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:51:43,823 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,43671,1733237501988, seqNum=-1] 2024-12-03T14:51:43,844 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:51:43,846 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:51:43,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0620 sec 2024-12-03T14:51:43,867 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733237503867, completionTime=-1 2024-12-03T14:51:43,870 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T14:51:43,870 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T14:51:43,895 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T14:51:43,896 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733237563896 2024-12-03T14:51:43,896 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733237623896 2024-12-03T14:51:43,896 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-12-03T14:51:43,898 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T14:51:43,906 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46647,1733237501335-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,906 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46647,1733237501335-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,906 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46647,1733237501335-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,908 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a5d22df9eca2:46647, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,908 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,909 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,916 DEBUG [master/a5d22df9eca2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T14:51:43,939 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.835sec 2024-12-03T14:51:43,940 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T14:51:43,942 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T14:51:43,943 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T14:51:43,943 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T14:51:43,943 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T14:51:43,944 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46647,1733237501335-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T14:51:43,945 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46647,1733237501335-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T14:51:43,949 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T14:51:43,950 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T14:51:43,951 INFO [master/a5d22df9eca2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a5d22df9eca2,46647,1733237501335-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:43,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24a6d0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:51:43,966 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a5d22df9eca2,46647,-1 for getting cluster id 2024-12-03T14:51:43,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T14:51:43,979 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1ef03b02-7d7a-4dc2-9ea3-fcb035dd7514' 2024-12-03T14:51:43,981 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T14:51:43,981 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1ef03b02-7d7a-4dc2-9ea3-fcb035dd7514" 2024-12-03T14:51:43,982 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a964ebd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:51:43,982 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a5d22df9eca2,46647,-1] 2024-12-03T14:51:43,984 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T14:51:43,987 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:51:43,987 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T14:51:43,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b80e06f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T14:51:43,990 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T14:51:43,997 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a5d22df9eca2,43671,1733237501988, seqNum=-1] 2024-12-03T14:51:43,998 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:51:44,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:51:44,017 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:44,018 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:38155/hbase 2024-12-03T14:51:44,032 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=5153 2024-12-03T14:51:44,048 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:44,052 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:44,053 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:44,058 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80235839, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-80235839, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:44,072 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80235839/hregion-80235839.1733237504059, exclude list is [], retry=0 2024-12-03T14:51:44,077 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:44,077 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:44,077 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:44,080 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80235839/hregion-80235839.1733237504059 2024-12-03T14:51:44,080 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:51:44,081 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 73bbc1fcc8c263b84222a2aa8be2644b, NAME => 'testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:51:44,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741838_1014 (size=64) 2024-12-03T14:51:44,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741838_1014 (size=64) 2024-12-03T14:51:44,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741838_1014 (size=64) 2024-12-03T14:51:44,095 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:44,097 INFO [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,099 INFO [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73bbc1fcc8c263b84222a2aa8be2644b columnFamilyName a 2024-12-03T14:51:44,099 DEBUG [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:44,100 INFO [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] regionserver.HStore(327): Store=73bbc1fcc8c263b84222a2aa8be2644b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:44,101 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,102 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,103 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,104 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,104 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,106 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,110 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:44,110 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 73bbc1fcc8c263b84222a2aa8be2644b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71120718, jitterRate=0.059781283140182495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:51:44,112 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 73bbc1fcc8c263b84222a2aa8be2644b: Writing region info on filesystem at 1733237504095Initializing all the Stores at 1733237504096 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237504096Cleaning up temporary data from old regions at 1733237504104 (+8 ms)Region opened successfully at 1733237504111 (+7 ms) 2024-12-03T14:51:44,112 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 73bbc1fcc8c263b84222a2aa8be2644b, disabling compactions & flushes 2024-12-03T14:51:44,112 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b. 2024-12-03T14:51:44,112 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b. 2024-12-03T14:51:44,112 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b. after waiting 0 ms 2024-12-03T14:51:44,112 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b. 2024-12-03T14:51:44,112 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b. 2024-12-03T14:51:44,113 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 73bbc1fcc8c263b84222a2aa8be2644b: Waiting for close lock at 1733237504112Disabling compacts and flushes for region at 1733237504112Disabling writes for close at 1733237504112Writing region close event to WAL at 1733237504112Closed at 1733237504112 2024-12-03T14:51:44,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741837_1013 (size=93) 2024-12-03T14:51:44,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741837_1013 (size=93) 2024-12-03T14:51:44,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741837_1013 (size=93) 2024-12-03T14:51:44,133 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:44,133 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80235839:(num 1733237504059) 2024-12-03T14:51:44,135 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T14:51:44,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741839_1015 (size=276) 2024-12-03T14:51:44,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741839_1015 (size=276) 2024-12-03T14:51:44,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741839_1015 (size=276) 2024-12-03T14:51:44,153 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T14:51:44,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741840_1016 (size=230) 2024-12-03T14:51:44,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741840_1016 (size=230) 2024-12-03T14:51:44,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741840_1016 (size=230) 2024-12-03T14:51:44,186 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1, size=276 (276bytes) 2024-12-03T14:51:44,187 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T14:51:44,187 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T14:51:44,187 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1 2024-12-03T14:51:44,191 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1 after 3ms 2024-12-03T14:51:44,196 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,197 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1 took 12ms 2024-12-03T14:51:44,201 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1 so closing down 2024-12-03T14:51:44,204 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T14:51:44,206 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:44,207 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T14:51:44,208 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741841_1017 (size=276) 2024-12-03T14:51:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741841_1017 (size=276) 2024-12-03T14:51:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741841_1017 (size=276) 2024-12-03T14:51:44,222 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T14:51:44,224 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002 2024-12-03T14:51:44,228 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-03T14:51:44,228 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1, journal: Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1, size=276 (276bytes) at 1733237504186Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1 so closing down at 1733237504201 (+15 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000001-wal-1.temp at 1733237504207 (+6 ms)3 split writer threads finished at 1733237504208 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733237504222 (+14 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002 at 1733237504225 (+3 ms)Processed 2 edits across 1 Regions in 27 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733237504228 (+3 ms) 2024-12-03T14:51:44,242 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2, size=230 (230bytes) 2024-12-03T14:51:44,242 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2 2024-12-03T14:51:44,243 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2 after 1ms 2024-12-03T14:51:44,247 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,247 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2 took 5ms 2024-12-03T14:51:44,249 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2 so closing down 2024-12-03T14:51:44,249 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:44,252 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T14:51:44,254 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T14:51:44,254 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:44,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741842_1018 (size=230) 2024-12-03T14:51:44,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741842_1018 (size=230) 2024-12-03T14:51:44,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741842_1018 (size=230) 2024-12-03T14:51:44,263 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T14:51:44,267 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,270 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-12-03T14:51:44,272 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-03T14:51:44,272 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2, journal: Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2, size=230 (230bytes) at 1733237504242Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2 so closing down at 1733237504249 (+7 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002-wal-2.temp at 1733237504254 (+5 ms)3 split writer threads finished at 1733237504254Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733237504263 (+9 ms)Processed 1 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733237504272 (+9 ms) 2024-12-03T14:51:44,272 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:44,274 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:44,288 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal.1733237504276, exclude list is [], retry=0 2024-12-03T14:51:44,292 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:44,293 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:44,293 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:44,295 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal.1733237504276 2024-12-03T14:51:44,295 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:51:44,295 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 73bbc1fcc8c263b84222a2aa8be2644b, NAME => 'testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:44,295 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:44,296 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,296 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,298 INFO [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,299 INFO [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 73bbc1fcc8c263b84222a2aa8be2644b columnFamilyName a 2024-12-03T14:51:44,299 DEBUG [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:44,300 INFO [StoreOpener-73bbc1fcc8c263b84222a2aa8be2644b-1 {}] regionserver.HStore(327): Store=73bbc1fcc8c263b84222a2aa8be2644b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:44,301 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,302 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,307 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002 2024-12-03T14:51:44,312 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,318 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002 2024-12-03T14:51:44,323 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 73bbc1fcc8c263b84222a2aa8be2644b 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T14:51:44,370 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/.tmp/a/fac47d2dbd8440ab95dc1e8c1af94b02 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733237504133/Put/seqid=0 2024-12-03T14:51:44,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741844_1020 (size=5170) 2024-12-03T14:51:44,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741844_1020 (size=5170) 2024-12-03T14:51:44,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741844_1020 (size=5170) 2024-12-03T14:51:44,384 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/.tmp/a/fac47d2dbd8440ab95dc1e8c1af94b02 2024-12-03T14:51:44,452 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/.tmp/a/fac47d2dbd8440ab95dc1e8c1af94b02 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/a/fac47d2dbd8440ab95dc1e8c1af94b02 2024-12-03T14:51:44,466 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/a/fac47d2dbd8440ab95dc1e8c1af94b02, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T14:51:44,471 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 73bbc1fcc8c263b84222a2aa8be2644b in 148ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T14:51:44,473 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/0000000000000000002 2024-12-03T14:51:44,474 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,474 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,477 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 73bbc1fcc8c263b84222a2aa8be2644b 2024-12-03T14:51:44,480 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/73bbc1fcc8c263b84222a2aa8be2644b/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T14:51:44,482 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 73bbc1fcc8c263b84222a2aa8be2644b; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67397115, jitterRate=0.004295274615287781}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:51:44,483 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 73bbc1fcc8c263b84222a2aa8be2644b: Writing region info on filesystem at 1733237504296Initializing all the Stores at 1733237504297 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237504297Obtaining lock to block concurrent updates at 1733237504323 (+26 ms)Preparing flush snapshotting stores in 73bbc1fcc8c263b84222a2aa8be2644b at 1733237504323Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733237504326 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733237504049.73bbc1fcc8c263b84222a2aa8be2644b. at 1733237504326Flushing 73bbc1fcc8c263b84222a2aa8be2644b/a: creating writer at 1733237504328 (+2 ms)Flushing 73bbc1fcc8c263b84222a2aa8be2644b/a: appending metadata at 1733237504361 (+33 ms)Flushing 73bbc1fcc8c263b84222a2aa8be2644b/a: closing flushed file at 1733237504363 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35fb8690: reopening flushed file at 1733237504450 (+87 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 73bbc1fcc8c263b84222a2aa8be2644b in 148ms, sequenceid=2, compaction requested=false; wal=null at 1733237504471 (+21 ms)Cleaning up temporary data from old regions at 1733237504474 (+3 ms)Region opened successfully at 1733237504483 (+9 ms) 2024-12-03T14:51:44,505 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=375 (was 363) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:45834 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:44512 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:57152 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:44654 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:57168 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:45690 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:38155/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=5067 (was 5153) 2024-12-03T14:51:44,515 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=5067 2024-12-03T14:51:44,533 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:44,535 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:44,536 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:44,539 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38829390, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-38829390, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:44,552 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38829390/hregion-38829390.1733237504540, exclude list is [], retry=0 2024-12-03T14:51:44,556 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:44,556 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:44,556 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:44,559 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38829390/hregion-38829390.1733237504540 2024-12-03T14:51:44,560 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:44,560 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 65c26f9bbde414d21ef6cac2823df6e0, NAME => 'testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:51:44,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741846_1022 (size=64) 2024-12-03T14:51:44,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741846_1022 (size=64) 2024-12-03T14:51:44,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741846_1022 (size=64) 2024-12-03T14:51:44,572 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:44,573 INFO [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,575 INFO [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65c26f9bbde414d21ef6cac2823df6e0 columnFamilyName a 2024-12-03T14:51:44,575 DEBUG [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:44,576 INFO [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] regionserver.HStore(327): Store=65c26f9bbde414d21ef6cac2823df6e0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:44,576 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,577 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,578 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,578 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,578 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,580 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,583 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:44,584 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 65c26f9bbde414d21ef6cac2823df6e0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72572116, jitterRate=0.0814087986946106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:51:44,584 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 65c26f9bbde414d21ef6cac2823df6e0: Writing region info on filesystem at 1733237504572Initializing all the Stores at 1733237504573 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237504573Cleaning up temporary data from old regions at 1733237504578 (+5 ms)Region opened successfully at 1733237504584 (+6 ms) 2024-12-03T14:51:44,584 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 65c26f9bbde414d21ef6cac2823df6e0, disabling compactions & flushes 2024-12-03T14:51:44,584 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0. 2024-12-03T14:51:44,584 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0. 2024-12-03T14:51:44,584 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0. after waiting 0 ms 2024-12-03T14:51:44,584 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0. 2024-12-03T14:51:44,584 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0. 2024-12-03T14:51:44,585 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 65c26f9bbde414d21ef6cac2823df6e0: Waiting for close lock at 1733237504584Disabling compacts and flushes for region at 1733237504584Disabling writes for close at 1733237504584Writing region close event to WAL at 1733237504584Closed at 1733237504584 2024-12-03T14:51:44,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741845_1021 (size=93) 2024-12-03T14:51:44,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741845_1021 (size=93) 2024-12-03T14:51:44,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741845_1021 (size=93) 2024-12-03T14:51:44,591 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:44,591 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38829390:(num 1733237504540) 2024-12-03T14:51:44,592 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T14:51:44,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741847_1023 (size=276) 2024-12-03T14:51:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741847_1023 (size=276) 2024-12-03T14:51:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741847_1023 (size=276) 2024-12-03T14:51:44,605 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T14:51:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741848_1024 (size=230) 2024-12-03T14:51:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741848_1024 (size=230) 2024-12-03T14:51:44,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741848_1024 (size=230) 2024-12-03T14:51:44,629 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2, size=230 (230bytes) 2024-12-03T14:51:44,629 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2 2024-12-03T14:51:44,630 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2 after 1ms 2024-12-03T14:51:44,634 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,634 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2 took 5ms 2024-12-03T14:51:44,637 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2 so closing down 2024-12-03T14:51:44,637 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:44,639 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T14:51:44,641 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T14:51:44,642 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:44,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741849_1025 (size=230) 2024-12-03T14:51:44,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741849_1025 (size=230) 2024-12-03T14:51:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741849_1025 (size=230) 2024-12-03T14:51:44,649 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T14:51:44,651 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002 2024-12-03T14:51:44,651 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-12-03T14:51:44,651 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2, journal: Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2, size=230 (230bytes) at 1733237504629Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2 so closing down at 1733237504637 (+8 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002-wal-2.temp at 1733237504641 (+4 ms)3 split writer threads finished at 1733237504642 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733237504649 (+7 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002 at 1733237504651 (+2 ms)Processed 1 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1733237504651 2024-12-03T14:51:44,666 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1, size=276 (276bytes) 2024-12-03T14:51:44,666 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1 2024-12-03T14:51:44,667 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1 after 0ms 2024-12-03T14:51:44,670 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,670 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1 took 4ms 2024-12-03T14:51:44,672 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1 so closing down 2024-12-03T14:51:44,673 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:44,674 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T14:51:44,676 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T14:51:44,676 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:44,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741850_1026 (size=276) 2024-12-03T14:51:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741850_1026 (size=276) 2024-12-03T14:51:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741850_1026 (size=276) 2024-12-03T14:51:44,685 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T14:51:44,688 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,691 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002, length=230 2024-12-03T14:51:44,693 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002 2024-12-03T14:51:44,693 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-12-03T14:51:44,693 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1, journal: Splitting hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1, size=276 (276bytes) at 1733237504666Finishing writing output for hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1 so closing down at 1733237504673 (+7 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000001-wal-1.temp at 1733237504676 (+3 ms)3 split writer threads finished at 1733237504676Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733237504685 (+9 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002 at 1733237504693 (+8 ms)Processed 2 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1733237504693 2024-12-03T14:51:44,693 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:44,696 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:44,707 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal.1733237504696, exclude list is [], retry=0 2024-12-03T14:51:44,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:44,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:44,711 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:44,714 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal.1733237504696 2024-12-03T14:51:44,715 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:44,715 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 65c26f9bbde414d21ef6cac2823df6e0, NAME => 'testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:44,715 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:44,715 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,715 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,717 INFO [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,719 INFO [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65c26f9bbde414d21ef6cac2823df6e0 columnFamilyName a 2024-12-03T14:51:44,719 DEBUG [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:44,720 INFO [StoreOpener-65c26f9bbde414d21ef6cac2823df6e0-1 {}] regionserver.HStore(327): Store=65c26f9bbde414d21ef6cac2823df6e0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:44,720 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,721 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,723 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,724 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002 2024-12-03T14:51:44,727 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:44,728 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002 2024-12-03T14:51:44,728 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 65c26f9bbde414d21ef6cac2823df6e0 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T14:51:44,743 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/.tmp/a/20648dcec24e48c8a917611b483d5081 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733237504591/Put/seqid=0 2024-12-03T14:51:44,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741852_1028 (size=5170) 2024-12-03T14:51:44,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741852_1028 (size=5170) 2024-12-03T14:51:44,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741852_1028 (size=5170) 2024-12-03T14:51:44,752 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/.tmp/a/20648dcec24e48c8a917611b483d5081 2024-12-03T14:51:44,761 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/.tmp/a/20648dcec24e48c8a917611b483d5081 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/a/20648dcec24e48c8a917611b483d5081 2024-12-03T14:51:44,770 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/a/20648dcec24e48c8a917611b483d5081, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T14:51:44,770 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 65c26f9bbde414d21ef6cac2823df6e0 in 42ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T14:51:44,771 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/0000000000000000002 2024-12-03T14:51:44,772 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,772 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,774 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 65c26f9bbde414d21ef6cac2823df6e0 2024-12-03T14:51:44,777 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/65c26f9bbde414d21ef6cac2823df6e0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T14:51:44,779 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 65c26f9bbde414d21ef6cac2823df6e0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68027310, jitterRate=0.013685911893844604}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:51:44,779 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 65c26f9bbde414d21ef6cac2823df6e0: Writing region info on filesystem at 1733237504715Initializing all the Stores at 1733237504717 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237504717Obtaining lock to block concurrent updates at 1733237504728 (+11 ms)Preparing flush snapshotting stores in 65c26f9bbde414d21ef6cac2823df6e0 at 1733237504728Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733237504729 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733237504533.65c26f9bbde414d21ef6cac2823df6e0. at 1733237504729Flushing 65c26f9bbde414d21ef6cac2823df6e0/a: creating writer at 1733237504729Flushing 65c26f9bbde414d21ef6cac2823df6e0/a: appending metadata at 1733237504742 (+13 ms)Flushing 65c26f9bbde414d21ef6cac2823df6e0/a: closing flushed file at 1733237504743 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b471f4: reopening flushed file at 1733237504760 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 65c26f9bbde414d21ef6cac2823df6e0 in 42ms, sequenceid=2, compaction requested=false; wal=null at 1733237504770 (+10 ms)Cleaning up temporary data from old regions at 1733237504772 (+2 ms)Region opened successfully at 1733237504779 (+7 ms) 2024-12-03T14:51:44,793 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:44512 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:57152 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:44736 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:45916 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:57224 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=130 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=5062 (was 5067) 2024-12-03T14:51:44,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=130, ProcessCount=11, AvailableMemoryMB=5061 2024-12-03T14:51:44,820 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:44,822 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:44,823 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:44,826 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-96983550, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-96983550, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:44,839 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-96983550/hregion-96983550.1733237504827, exclude list is [], retry=0 2024-12-03T14:51:44,842 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:44,843 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:44,843 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:44,845 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-96983550/hregion-96983550.1733237504827 2024-12-03T14:51:44,846 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:51:44,846 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a192060dfbf4a2ed9e288a59303e8a94, NAME => 'testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:51:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741854_1030 (size=64) 2024-12-03T14:51:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741854_1030 (size=64) 2024-12-03T14:51:44,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741854_1030 (size=64) 2024-12-03T14:51:44,858 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:44,859 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,861 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a192060dfbf4a2ed9e288a59303e8a94 columnFamilyName a 2024-12-03T14:51:44,861 DEBUG [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:44,862 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(327): Store=a192060dfbf4a2ed9e288a59303e8a94/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:44,862 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,864 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a192060dfbf4a2ed9e288a59303e8a94 columnFamilyName b 2024-12-03T14:51:44,864 DEBUG [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:44,864 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(327): Store=a192060dfbf4a2ed9e288a59303e8a94/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:44,864 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,866 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a192060dfbf4a2ed9e288a59303e8a94 columnFamilyName c 2024-12-03T14:51:44,866 DEBUG [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:44,867 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(327): Store=a192060dfbf4a2ed9e288a59303e8a94/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:44,867 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,868 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,870 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,870 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,870 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:51:44,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:44,875 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:44,875 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a192060dfbf4a2ed9e288a59303e8a94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73598017, jitterRate=0.0966959148645401}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:51:44,876 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a192060dfbf4a2ed9e288a59303e8a94: Writing region info on filesystem at 1733237504858Initializing all the Stores at 1733237504859 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237504859Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237504859Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237504859Cleaning up temporary data from old regions at 1733237504870 (+11 ms)Region opened successfully at 1733237504875 (+5 ms) 2024-12-03T14:51:44,876 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a192060dfbf4a2ed9e288a59303e8a94, disabling compactions & flushes 2024-12-03T14:51:44,876 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:44,876 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:44,876 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. after waiting 0 ms 2024-12-03T14:51:44,876 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:44,876 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:44,876 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a192060dfbf4a2ed9e288a59303e8a94: Waiting for close lock at 1733237504876Disabling compacts and flushes for region at 1733237504876Disabling writes for close at 1733237504876Writing region close event to WAL at 1733237504876Closed at 1733237504876 2024-12-03T14:51:44,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741853_1029 (size=93) 2024-12-03T14:51:44,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741853_1029 (size=93) 2024-12-03T14:51:44,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741853_1029 (size=93) 2024-12-03T14:51:44,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:44,883 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-96983550:(num 1733237504827) 2024-12-03T14:51:44,883 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:44,885 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:44,898 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886, exclude list is [], retry=0 2024-12-03T14:51:44,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:44,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:44,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:44,904 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 2024-12-03T14:51:44,905 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:45,140 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886, size=0 (0bytes) 2024-12-03T14:51:45,140 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 might be still open, length is 0 2024-12-03T14:51:45,140 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 2024-12-03T14:51:45,142 WARN [IPC Server handler 4 on default port 38155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-03T14:51:45,142 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 after 2ms 2024-12-03T14:51:46,297 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:45946 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37353:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45946 dst: /127.0.0.1:37353 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37353 remote=/127.0.0.1:45946]. Total timeout mills is 60000, 58803 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:46,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:57252 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57252 dst: /127.0.0.1:40191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:46,298 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:44770 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38931:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44770 dst: /127.0.0.1:38931 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:46,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741855_1032 (size=200589) 2024-12-03T14:51:46,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741855_1032 (size=200589) 2024-12-03T14:51:46,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741855_1032 (size=200589) 2024-12-03T14:51:49,144 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 after 4004ms 2024-12-03T14:51:49,155 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:49,157 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 took 4017ms 2024-12-03T14:51:49,162 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733237504886.temp 2024-12-03T14:51:49,165 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000000001-wal.1733237504886.temp 2024-12-03T14:51:49,236 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:51:49,264 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886; continuing. 2024-12-03T14:51:49,264 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 so closing down 2024-12-03T14:51:49,264 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:49,264 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:49,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741856_1033 (size=200597) 2024-12-03T14:51:49,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741856_1033 (size=200597) 2024-12-03T14:51:49,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741856_1033 (size=200597) 2024-12-03T14:51:49,269 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000000001-wal.1733237504886.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-12-03T14:51:49,271 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000000001-wal.1733237504886.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000003002 2024-12-03T14:51:49,272 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 115 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T14:51:49,272 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886, journal: Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886, size=0 (0bytes) at 1733237505140Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000000001-wal.1733237504886.temp at 1733237509165 (+4025 ms)Split 1024 edits, skipped 0 edits. at 1733237509214 (+49 ms)Split 2048 edits, skipped 0 edits. at 1733237509243 (+29 ms)Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 so closing down at 1733237509264 (+21 ms)3 split writer threads finished at 1733237509264Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000000001-wal.1733237504886.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1733237509270 (+6 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000000001-wal.1733237504886.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000003002 at 1733237509271 (+1 ms)Processed 3002 edits across 1 Regions in 115 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886, size=0, length=0, corrupted=false, cancelled=false at 1733237509272 (+1 ms) 2024-12-03T14:51:49,274 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237504886 2024-12-03T14:51:49,276 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000003002 2024-12-03T14:51:49,276 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:49,279 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:49,293 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237509279, exclude list is [], retry=0 2024-12-03T14:51:49,296 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:49,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:49,297 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:49,304 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237509279 2024-12-03T14:51:49,304 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:49,304 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:49,309 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:49,311 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a192060dfbf4a2ed9e288a59303e8a94 columnFamilyName a 2024-12-03T14:51:49,311 DEBUG [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:49,311 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(327): Store=a192060dfbf4a2ed9e288a59303e8a94/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:49,312 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:49,313 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a192060dfbf4a2ed9e288a59303e8a94 columnFamilyName b 2024-12-03T14:51:49,313 DEBUG [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:49,314 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(327): Store=a192060dfbf4a2ed9e288a59303e8a94/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:49,314 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:49,315 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a192060dfbf4a2ed9e288a59303e8a94 columnFamilyName c 2024-12-03T14:51:49,315 DEBUG [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:49,316 INFO [StoreOpener-a192060dfbf4a2ed9e288a59303e8a94-1 {}] regionserver.HStore(327): Store=a192060dfbf4a2ed9e288a59303e8a94/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:49,316 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:49,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:49,319 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:49,320 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000003002 2024-12-03T14:51:49,324 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:49,332 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T14:51:49,373 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T14:51:49,718 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a192060dfbf4a2ed9e288a59303e8a94 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T14:51:49,767 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/a/27f10939dccc4fc89100b8862bed37c2 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733237504917/Put/seqid=0 2024-12-03T14:51:49,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741858_1035 (size=50463) 2024-12-03T14:51:49,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741858_1035 (size=50463) 2024-12-03T14:51:49,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741858_1035 (size=50463) 2024-12-03T14:51:49,776 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/a/27f10939dccc4fc89100b8862bed37c2 2024-12-03T14:51:49,784 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/a/27f10939dccc4fc89100b8862bed37c2 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/a/27f10939dccc4fc89100b8862bed37c2 2024-12-03T14:51:49,791 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/a/27f10939dccc4fc89100b8862bed37c2, entries=754, sequenceid=754, filesize=49.3 K 2024-12-03T14:51:49,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for a192060dfbf4a2ed9e288a59303e8a94 in 73ms, sequenceid=754, compaction requested=false; wal=null 2024-12-03T14:51:49,810 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T14:51:49,811 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a192060dfbf4a2ed9e288a59303e8a94 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T14:51:49,820 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/a/f17a9db8bc1348deae85927bc01d875d is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733237504978/Put/seqid=0 2024-12-03T14:51:49,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741859_1036 (size=20072) 2024-12-03T14:51:49,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741859_1036 (size=20072) 2024-12-03T14:51:49,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741859_1036 (size=20072) 2024-12-03T14:51:49,831 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/a/f17a9db8bc1348deae85927bc01d875d 2024-12-03T14:51:49,854 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/b/5973bb2cde8e4d50a2745596116326ed is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733237505000/Put/seqid=0 2024-12-03T14:51:49,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741860_1037 (size=35835) 2024-12-03T14:51:49,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741860_1037 (size=35835) 2024-12-03T14:51:49,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741860_1037 (size=35835) 2024-12-03T14:51:50,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/b/5973bb2cde8e4d50a2745596116326ed 2024-12-03T14:51:50,276 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/a/f17a9db8bc1348deae85927bc01d875d as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/a/f17a9db8bc1348deae85927bc01d875d 2024-12-03T14:51:50,285 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/a/f17a9db8bc1348deae85927bc01d875d, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-03T14:51:50,287 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/b/5973bb2cde8e4d50a2745596116326ed as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/b/5973bb2cde8e4d50a2745596116326ed 2024-12-03T14:51:50,294 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/b/5973bb2cde8e4d50a2745596116326ed, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-03T14:51:50,294 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a192060dfbf4a2ed9e288a59303e8a94 in 483ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-03T14:51:50,312 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T14:51:50,312 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a192060dfbf4a2ed9e288a59303e8a94 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T14:51:50,319 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/b/d317ba55dc6348d18b1d82951d56cca2 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733237505021/Put/seqid=0 2024-12-03T14:51:50,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741861_1038 (size=35082) 2024-12-03T14:51:50,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741861_1038 (size=35082) 2024-12-03T14:51:50,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741861_1038 (size=35082) 2024-12-03T14:51:50,330 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/b/d317ba55dc6348d18b1d82951d56cca2 2024-12-03T14:51:50,354 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/c/b14cd11c66a64bc0aef82dff9f5f4280 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733237505059/Put/seqid=0 2024-12-03T14:51:50,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741862_1039 (size=20825) 2024-12-03T14:51:50,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741862_1039 (size=20825) 2024-12-03T14:51:50,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741862_1039 (size=20825) 2024-12-03T14:51:50,362 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/c/b14cd11c66a64bc0aef82dff9f5f4280 2024-12-03T14:51:50,372 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/b/d317ba55dc6348d18b1d82951d56cca2 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/b/d317ba55dc6348d18b1d82951d56cca2 2024-12-03T14:51:50,380 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/b/d317ba55dc6348d18b1d82951d56cca2, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-03T14:51:50,382 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/c/b14cd11c66a64bc0aef82dff9f5f4280 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/c/b14cd11c66a64bc0aef82dff9f5f4280 2024-12-03T14:51:50,389 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/c/b14cd11c66a64bc0aef82dff9f5f4280, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-03T14:51:50,390 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for a192060dfbf4a2ed9e288a59303e8a94 in 78ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-03T14:51:50,399 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733237505097/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:50,403 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000003002 2024-12-03T14:51:50,403 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T14:51:50,404 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a192060dfbf4a2ed9e288a59303e8a94 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-03T14:51:50,413 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/c/388a4f38b6114a66a989ada8d5965222 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733237505066/Put/seqid=0 2024-12-03T14:51:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741863_1040 (size=50301) 2024-12-03T14:51:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741863_1040 (size=50301) 2024-12-03T14:51:50,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741863_1040 (size=50301) 2024-12-03T14:51:50,825 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/c/388a4f38b6114a66a989ada8d5965222 2024-12-03T14:51:50,835 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 388a4f38b6114a66a989ada8d5965222 2024-12-03T14:51:50,837 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/.tmp/c/388a4f38b6114a66a989ada8d5965222 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/c/388a4f38b6114a66a989ada8d5965222 2024-12-03T14:51:50,844 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 388a4f38b6114a66a989ada8d5965222 2024-12-03T14:51:50,845 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/c/388a4f38b6114a66a989ada8d5965222, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-03T14:51:50,845 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for a192060dfbf4a2ed9e288a59303e8a94 in 442ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-03T14:51:50,846 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/0000000000000003002 2024-12-03T14:51:50,847 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:50,847 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:50,847 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T14:51:50,849 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a192060dfbf4a2ed9e288a59303e8a94 2024-12-03T14:51:50,852 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenIntoWAL/a192060dfbf4a2ed9e288a59303e8a94/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-03T14:51:50,853 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a192060dfbf4a2ed9e288a59303e8a94; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72303245, jitterRate=0.07740230858325958}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T14:51:50,853 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a192060dfbf4a2ed9e288a59303e8a94: Writing region info on filesystem at 1733237509305Initializing all the Stores at 1733237509306 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237509306Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237509309 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237509309Cleaning up temporary data from old regions at 1733237510847 (+1538 ms)Region opened successfully at 1733237510853 (+6 ms) 2024-12-03T14:51:50,927 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a192060dfbf4a2ed9e288a59303e8a94, disabling compactions & flushes 2024-12-03T14:51:50,927 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:50,927 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:50,927 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. after waiting 0 ms 2024-12-03T14:51:50,927 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:50,929 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733237504821.a192060dfbf4a2ed9e288a59303e8a94. 2024-12-03T14:51:50,929 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a192060dfbf4a2ed9e288a59303e8a94: Waiting for close lock at 1733237510927Disabling compacts and flushes for region at 1733237510927Disabling writes for close at 1733237510927Writing region close event to WAL at 1733237510929 (+2 ms)Closed at 1733237510929 2024-12-03T14:51:50,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741857_1034 (size=93) 2024-12-03T14:51:50,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741857_1034 (size=93) 2024-12-03T14:51:50,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741857_1034 (size=93) 2024-12-03T14:51:50,938 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:50,938 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733237509279) 2024-12-03T14:51:50,955 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=401 (was 385) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1996423001_22 at /127.0.0.1:51030 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1996423001_22 at /127.0.0.1:49294 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46259 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:46259 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:38155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1996423001_22 at /127.0.0.1:49246 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:32851 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:38155 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1996423001_22 at /127.0.0.1:47810 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32851 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=127 (was 130), ProcessCount=11 (was 11), AvailableMemoryMB=4995 (was 5061) 2024-12-03T14:51:50,968 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=127, ProcessCount=11, AvailableMemoryMB=4991 2024-12-03T14:51:50,982 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:50,984 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:50,985 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:50,988 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89903543, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-89903543, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:51,000 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89903543/hregion-89903543.1733237510989, exclude list is [], retry=0 2024-12-03T14:51:51,004 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:51,004 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:51,005 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:51,007 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89903543/hregion-89903543.1733237510989 2024-12-03T14:51:51,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:51,009 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e13c1c468864a4a4b030f3f42f3ab6e3, NAME => 'test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:51:51,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741865_1042 (size=43) 2024-12-03T14:51:51,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741865_1042 (size=43) 2024-12-03T14:51:51,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741865_1042 (size=43) 2024-12-03T14:51:51,025 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:51,026 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,028 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e13c1c468864a4a4b030f3f42f3ab6e3 columnFamilyName a 2024-12-03T14:51:51,028 DEBUG [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:51,029 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(327): Store=e13c1c468864a4a4b030f3f42f3ab6e3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:51,029 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,031 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e13c1c468864a4a4b030f3f42f3ab6e3 columnFamilyName b 2024-12-03T14:51:51,031 DEBUG [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:51,031 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(327): Store=e13c1c468864a4a4b030f3f42f3ab6e3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:51,031 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,033 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e13c1c468864a4a4b030f3f42f3ab6e3 columnFamilyName c 2024-12-03T14:51:51,033 DEBUG [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:51,033 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(327): Store=e13c1c468864a4a4b030f3f42f3ab6e3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,035 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,036 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,036 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,037 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:51:51,038 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,041 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:51,041 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e13c1c468864a4a4b030f3f42f3ab6e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67586235, jitterRate=0.0071133822202682495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:51:51,043 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e13c1c468864a4a4b030f3f42f3ab6e3: Writing region info on filesystem at 1733237511025Initializing all the Stores at 1733237511026 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237511026Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237511026Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237511026Cleaning up temporary data from old regions at 1733237511036 (+10 ms)Region opened successfully at 1733237511043 (+7 ms) 2024-12-03T14:51:51,043 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e13c1c468864a4a4b030f3f42f3ab6e3, disabling compactions & flushes 2024-12-03T14:51:51,043 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:51,043 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:51,043 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. after waiting 0 ms 2024-12-03T14:51:51,043 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:51,044 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:51,044 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e13c1c468864a4a4b030f3f42f3ab6e3: Waiting for close lock at 1733237511043Disabling compacts and flushes for region at 1733237511043Disabling writes for close at 1733237511043Writing region close event to WAL at 1733237511044 (+1 ms)Closed at 1733237511044 2024-12-03T14:51:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741864_1041 (size=93) 2024-12-03T14:51:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741864_1041 (size=93) 2024-12-03T14:51:51,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741864_1041 (size=93) 2024-12-03T14:51:51,050 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:51,050 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89903543:(num 1733237510989) 2024-12-03T14:51:51,051 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:51,053 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:51,064 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053, exclude list is [], retry=0 2024-12-03T14:51:51,067 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:51,067 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:51,068 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:51,070 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053 2024-12-03T14:51:51,070 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:51,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741866_1043 (size=200357) 2024-12-03T14:51:51,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741866_1043 (size=200357) 2024-12-03T14:51:51,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741866_1043 (size=200357) 2024-12-03T14:51:51,298 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053, size=195.7 K (200357bytes) 2024-12-03T14:51:51,298 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053 2024-12-03T14:51:51,299 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053 after 1ms 2024-12-03T14:51:51,303 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:51,305 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053 took 7ms 2024-12-03T14:51:51,310 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733237511053.temp 2024-12-03T14:51:51,314 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000000001-wal.1733237511053.temp 2024-12-03T14:51:51,353 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053 so closing down 2024-12-03T14:51:51,353 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:51,354 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:51,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741867_1044 (size=200357) 2024-12-03T14:51:51,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741867_1044 (size=200357) 2024-12-03T14:51:51,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741867_1044 (size=200357) 2024-12-03T14:51:51,358 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000000001-wal.1733237511053.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-12-03T14:51:51,360 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000000001-wal.1733237511053.temp to hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003000 2024-12-03T14:51:51,360 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 55 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-12-03T14:51:51,360 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053, journal: Splitting hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053, size=195.7 K (200357bytes) at 1733237511298Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000000001-wal.1733237511053.temp at 1733237511314 (+16 ms)Split 1024 edits, skipped 0 edits. at 1733237511323 (+9 ms)Split 2048 edits, skipped 0 edits. at 1733237511338 (+15 ms)Finishing writing output for hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053 so closing down at 1733237511353 (+15 ms)3 split writer threads finished at 1733237511354 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000000001-wal.1733237511053.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1733237511358 (+4 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000000001-wal.1733237511053.temp to hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003000 at 1733237511360 (+2 ms)Processed 3000 edits across 1 Regions in 55 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1733237511360 2024-12-03T14:51:51,363 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511053 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237511053 2024-12-03T14:51:51,364 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003000 2024-12-03T14:51:51,364 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:51,367 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:51,385 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367, exclude list is [], retry=0 2024-12-03T14:51:51,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:51,389 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:51,390 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:51,391 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367 2024-12-03T14:51:51,392 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:51,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741868_1045 (size=200484) 2024-12-03T14:51:51,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741868_1045 (size=200484) 2024-12-03T14:51:51,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741868_1045 (size=200484) 2024-12-03T14:51:51,575 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367, size=195.8 K (200484bytes) 2024-12-03T14:51:51,575 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367 2024-12-03T14:51:51,576 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367 after 1ms 2024-12-03T14:51:51,580 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:51,581 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367 took 6ms 2024-12-03T14:51:51,586 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733237511367.temp 2024-12-03T14:51:51,588 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003001-wal.1733237511367.temp 2024-12-03T14:51:51,636 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367 so closing down 2024-12-03T14:51:51,636 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:51,636 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741869_1046 (size=200484) 2024-12-03T14:51:51,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741869_1046 (size=200484) 2024-12-03T14:51:51,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741869_1046 (size=200484) 2024-12-03T14:51:51,640 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003001-wal.1733237511367.temp (wrote 3000 edits, skipped 0 edits in 17 ms) 2024-12-03T14:51:51,642 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003001-wal.1733237511367.temp to hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000006000 2024-12-03T14:51:51,643 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-12-03T14:51:51,643 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367, journal: Splitting hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367, size=195.8 K (200484bytes) at 1733237511575Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003001-wal.1733237511367.temp at 1733237511588 (+13 ms)Split 1024 edits, skipped 0 edits. at 1733237511598 (+10 ms)Split 2048 edits, skipped 0 edits. at 1733237511617 (+19 ms)Finishing writing output for hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367 so closing down at 1733237511636 (+19 ms)3 split writer threads finished at 1733237511637 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003001-wal.1733237511367.temp (wrote 3000 edits, skipped 0 edits in 17 ms) at 1733237511641 (+4 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003001-wal.1733237511367.temp to hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000006000 at 1733237511642 (+1 ms)Processed 3000 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1733237511643 (+1 ms) 2024-12-03T14:51:51,645 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511367 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237511367 2024-12-03T14:51:51,647 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000006000 2024-12-03T14:51:51,647 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:51,650 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/test2727-manual,16010,1733237510981, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:51,660 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T14:51:51,661 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T14:51:51,663 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T14:51:51,663 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T14:51:51,663 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T14:51:51,663 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T14:51:51,665 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-03T14:51:51,665 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-03T14:51:51,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511650, exclude list is [], retry=0 2024-12-03T14:51:51,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:51,675 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:51,676 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:51,679 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733237510981/wal.1733237511650 2024-12-03T14:51:51,680 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:51,680 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e13c1c468864a4a4b030f3f42f3ab6e3, NAME => 'test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:51,680 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:51,680 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,680 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,682 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,684 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e13c1c468864a4a4b030f3f42f3ab6e3 columnFamilyName a 2024-12-03T14:51:51,684 DEBUG [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:51,685 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(327): Store=e13c1c468864a4a4b030f3f42f3ab6e3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:51,685 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,686 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e13c1c468864a4a4b030f3f42f3ab6e3 columnFamilyName b 2024-12-03T14:51:51,686 DEBUG [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:51,687 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(327): Store=e13c1c468864a4a4b030f3f42f3ab6e3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:51,687 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,689 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e13c1c468864a4a4b030f3f42f3ab6e3 columnFamilyName c 2024-12-03T14:51:51,689 DEBUG [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:51,690 INFO [StoreOpener-e13c1c468864a4a4b030f3f42f3ab6e3-1 {}] regionserver.HStore(327): Store=e13c1c468864a4a4b030f3f42f3ab6e3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:51,690 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:51,693 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003000 2024-12-03T14:51:51,696 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:51,745 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003000 2024-12-03T14:51:51,746 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000006000 2024-12-03T14:51:51,749 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:51,793 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000006000 2024-12-03T14:51:51,793 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e13c1c468864a4a4b030f3f42f3ab6e3 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-03T14:51:51,811 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/a/25f1d92c79884b89a8580dedc177adf4 is 41, key is test2727/a:100/1733237511397/Put/seqid=0 2024-12-03T14:51:51,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741871_1048 (size=84227) 2024-12-03T14:51:51,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741871_1048 (size=84227) 2024-12-03T14:51:51,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741871_1048 (size=84227) 2024-12-03T14:51:51,822 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/a/25f1d92c79884b89a8580dedc177adf4 2024-12-03T14:51:51,849 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/b/48af9df476204d4ab72b467a387b85cd is 41, key is test2727/b:100/1733237511439/Put/seqid=0 2024-12-03T14:51:51,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741872_1049 (size=84609) 2024-12-03T14:51:51,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741872_1049 (size=84609) 2024-12-03T14:51:51,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741872_1049 (size=84609) 2024-12-03T14:51:52,255 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/b/48af9df476204d4ab72b467a387b85cd 2024-12-03T14:51:52,281 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/c/397669a0602846569348b02dd5d714e9 is 41, key is test2727/c:100/1733237511484/Put/seqid=0 2024-12-03T14:51:52,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741873_1050 (size=84609) 2024-12-03T14:51:52,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741873_1050 (size=84609) 2024-12-03T14:51:52,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741873_1050 (size=84609) 2024-12-03T14:51:52,288 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/c/397669a0602846569348b02dd5d714e9 2024-12-03T14:51:52,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/a/25f1d92c79884b89a8580dedc177adf4 as hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/a/25f1d92c79884b89a8580dedc177adf4 2024-12-03T14:51:52,301 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/a/25f1d92c79884b89a8580dedc177adf4, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-03T14:51:52,302 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/b/48af9df476204d4ab72b467a387b85cd as hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/b/48af9df476204d4ab72b467a387b85cd 2024-12-03T14:51:52,309 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/b/48af9df476204d4ab72b467a387b85cd, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T14:51:52,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/.tmp/c/397669a0602846569348b02dd5d714e9 as hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/c/397669a0602846569348b02dd5d714e9 2024-12-03T14:51:52,318 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/c/397669a0602846569348b02dd5d714e9, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T14:51:52,318 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e13c1c468864a4a4b030f3f42f3ab6e3 in 525ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-03T14:51:52,319 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000003000 2024-12-03T14:51:52,319 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/0000000000000006000 2024-12-03T14:51:52,321 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:52,321 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:52,321 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:51:52,323 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e13c1c468864a4a4b030f3f42f3ab6e3 2024-12-03T14:51:52,326 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/test2727/e13c1c468864a4a4b030f3f42f3ab6e3/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-03T14:51:52,327 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e13c1c468864a4a4b030f3f42f3ab6e3; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74991937, jitterRate=0.11746694147586823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:51:52,328 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e13c1c468864a4a4b030f3f42f3ab6e3: Writing region info on filesystem at 1733237511680Initializing all the Stores at 1733237511682 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237511682Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237511682Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237511682Obtaining lock to block concurrent updates at 1733237511793 (+111 ms)Preparing flush snapshotting stores in e13c1c468864a4a4b030f3f42f3ab6e3 at 1733237511793Finished memstore snapshotting test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733237511793Flushing stores of test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. at 1733237511793Flushing e13c1c468864a4a4b030f3f42f3ab6e3/a: creating writer at 1733237511793Flushing e13c1c468864a4a4b030f3f42f3ab6e3/a: appending metadata at 1733237511810 (+17 ms)Flushing e13c1c468864a4a4b030f3f42f3ab6e3/a: closing flushed file at 1733237511810Flushing e13c1c468864a4a4b030f3f42f3ab6e3/b: creating writer at 1733237511828 (+18 ms)Flushing e13c1c468864a4a4b030f3f42f3ab6e3/b: appending metadata at 1733237511847 (+19 ms)Flushing e13c1c468864a4a4b030f3f42f3ab6e3/b: closing flushed file at 1733237511847Flushing e13c1c468864a4a4b030f3f42f3ab6e3/c: creating writer at 1733237512264 (+417 ms)Flushing e13c1c468864a4a4b030f3f42f3ab6e3/c: appending metadata at 1733237512280 (+16 ms)Flushing e13c1c468864a4a4b030f3f42f3ab6e3/c: closing flushed file at 1733237512280Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5242fd40: reopening flushed file at 1733237512294 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f6c3376: reopening flushed file at 1733237512301 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e6b3237: reopening flushed file at 1733237512310 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e13c1c468864a4a4b030f3f42f3ab6e3 in 525ms, sequenceid=6000, compaction requested=false; wal=null at 1733237512318 (+8 ms)Cleaning up temporary data from old regions at 1733237512321 (+3 ms)Region opened successfully at 1733237512328 (+7 ms) 2024-12-03T14:51:52,329 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-03T14:51:52,329 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e13c1c468864a4a4b030f3f42f3ab6e3, disabling compactions & flushes 2024-12-03T14:51:52,329 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:52,329 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:52,329 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. after waiting 0 ms 2024-12-03T14:51:52,329 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:52,330 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733237510983.e13c1c468864a4a4b030f3f42f3ab6e3. 2024-12-03T14:51:52,330 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e13c1c468864a4a4b030f3f42f3ab6e3: Waiting for close lock at 1733237512329Disabling compacts and flushes for region at 1733237512329Disabling writes for close at 1733237512329Writing region close event to WAL at 1733237512330 (+1 ms)Closed at 1733237512330 2024-12-03T14:51:52,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741870_1047 (size=93) 2024-12-03T14:51:52,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741870_1047 (size=93) 2024-12-03T14:51:52,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741870_1047 (size=93) 2024-12-03T14:51:52,337 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:52,337 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733237511650) 2024-12-03T14:51:52,351 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=403 (was 401) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:49246 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:47936 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:51154 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=149 (was 127) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4836 (was 4991) 2024-12-03T14:51:52,363 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=149, ProcessCount=11, AvailableMemoryMB=4835 2024-12-03T14:51:52,381 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:52,388 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:52,389 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733237512389 2024-12-03T14:51:52,395 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389 2024-12-03T14:51:52,397 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:52,399 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f35adadc81c25c753a782ec3d9c3dad, NAME => 'testSequentialEditLogSeqNum,,1733237512382.8f35adadc81c25c753a782ec3d9c3dad.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:52,399 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733237512382.8f35adadc81c25c753a782ec3d9c3dad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:52,399 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,399 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,400 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad doesn't exist for region: 8f35adadc81c25c753a782ec3d9c3dad on table testSequentialEditLogSeqNum 2024-12-03T14:51:52,401 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 8f35adadc81c25c753a782ec3d9c3dad on table testSequentialEditLogSeqNum 2024-12-03T14:51:52,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741875_1052 (size=62) 2024-12-03T14:51:52,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741875_1052 (size=62) 2024-12-03T14:51:52,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741875_1052 (size=62) 2024-12-03T14:51:52,413 INFO [StoreOpener-8f35adadc81c25c753a782ec3d9c3dad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,415 INFO [StoreOpener-8f35adadc81c25c753a782ec3d9c3dad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f35adadc81c25c753a782ec3d9c3dad columnFamilyName a 2024-12-03T14:51:52,415 DEBUG [StoreOpener-8f35adadc81c25c753a782ec3d9c3dad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:52,415 INFO [StoreOpener-8f35adadc81c25c753a782ec3d9c3dad-1 {}] regionserver.HStore(327): Store=8f35adadc81c25c753a782ec3d9c3dad/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:52,416 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,416 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,417 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,417 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,418 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8f35adadc81c25c753a782ec3d9c3dad 2024-12-03T14:51:52,420 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:52,421 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8f35adadc81c25c753a782ec3d9c3dad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67621947, jitterRate=0.007645532488822937}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:51:52,422 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8f35adadc81c25c753a782ec3d9c3dad: Writing region info on filesystem at 1733237512399Initializing all the Stores at 1733237512413 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237512413Cleaning up temporary data from old regions at 1733237512417 (+4 ms)Region opened successfully at 1733237512422 (+5 ms) 2024-12-03T14:51:52,435 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8f35adadc81c25c753a782ec3d9c3dad 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-03T14:51:52,456 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/.tmp/a/ff9639aa2d6e4467aa833716cce9f31f is 81, key is testSequentialEditLogSeqNum/a:x0/1733237512422/Put/seqid=0 2024-12-03T14:51:52,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741876_1053 (size=5833) 2024-12-03T14:51:52,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741876_1053 (size=5833) 2024-12-03T14:51:52,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741876_1053 (size=5833) 2024-12-03T14:51:52,464 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/.tmp/a/ff9639aa2d6e4467aa833716cce9f31f 2024-12-03T14:51:52,471 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/.tmp/a/ff9639aa2d6e4467aa833716cce9f31f as hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/a/ff9639aa2d6e4467aa833716cce9f31f 2024-12-03T14:51:52,478 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/a/ff9639aa2d6e4467aa833716cce9f31f, entries=10, sequenceid=13, filesize=5.7 K 2024-12-03T14:51:52,479 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8f35adadc81c25c753a782ec3d9c3dad in 44ms, sequenceid=13, compaction requested=false 2024-12-03T14:51:52,480 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8f35adadc81c25c753a782ec3d9c3dad: 2024-12-03T14:51:52,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T14:51:52,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T14:51:52,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T14:51:52,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T14:51:52,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T14:51:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741874_1051 (size=1616) 2024-12-03T14:51:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741874_1051 (size=1616) 2024-12-03T14:51:52,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741874_1051 (size=1616) 2024-12-03T14:51:52,513 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389, size=1.6 K (1616bytes) 2024-12-03T14:51:52,513 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389 2024-12-03T14:51:52,514 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389 after 1ms 2024-12-03T14:51:52,517 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:52,517 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389 took 4ms 2024-12-03T14:51:52,520 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389 so closing down 2024-12-03T14:51:52,520 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:52,521 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733237512389.temp 2024-12-03T14:51:52,523 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000003-wal.1733237512389.temp 2024-12-03T14:51:52,523 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:52,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741877_1054 (size=1175) 2024-12-03T14:51:52,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741877_1054 (size=1175) 2024-12-03T14:51:52,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741877_1054 (size=1175) 2024-12-03T14:51:52,536 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000003-wal.1733237512389.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T14:51:52,538 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000003-wal.1733237512389.temp to hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000020 2024-12-03T14:51:52,538 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-12-03T14:51:52,538 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389, journal: Splitting hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389, size=1.6 K (1616bytes) at 1733237512513Finishing writing output for hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389 so closing down at 1733237512520 (+7 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000003-wal.1733237512389.temp at 1733237512523 (+3 ms)3 split writer threads finished at 1733237512523Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000003-wal.1733237512389.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733237512536 (+13 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000003-wal.1733237512389.temp to hdfs://localhost:38155/hbase/data/default/testSequentialEditLogSeqNum/8f35adadc81c25c753a782ec3d9c3dad/recovered.edits/0000000000000000020 at 1733237512538 (+2 ms)Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:38155/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733237512380/wal.1733237512389, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1733237512538 2024-12-03T14:51:52,557 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:49246 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:51154 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=149 (was 149), ProcessCount=11 (was 11), AvailableMemoryMB=4799 (was 4835) 2024-12-03T14:51:52,571 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=149, ProcessCount=11, AvailableMemoryMB=4797 2024-12-03T14:51:52,595 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:52,598 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:52,622 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:51:52,625 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-00545329, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-00545329, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:52,644 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-00545329/hregion-00545329.1733237512626, exclude list is [], retry=0 2024-12-03T14:51:52,648 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:52,648 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:52,649 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:52,652 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-00545329/hregion-00545329.1733237512626 2024-12-03T14:51:52,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:52,653 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b9b4e0a5e7cb9efb1cb8880f73ac6e4d, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:51:52,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741879_1056 (size=70) 2024-12-03T14:51:52,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741879_1056 (size=70) 2024-12-03T14:51:52,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741879_1056 (size=70) 2024-12-03T14:51:52,670 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:52,673 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,675 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName a 2024-12-03T14:51:52,675 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:52,676 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:52,676 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,678 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName b 2024-12-03T14:51:52,678 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:52,678 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:52,678 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,680 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName c 2024-12-03T14:51:52,680 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:52,681 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:52,681 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,682 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,682 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,683 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,684 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,684 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:51:52,686 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,689 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:52,689 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9b4e0a5e7cb9efb1cb8880f73ac6e4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62701964, jitterRate=-0.06566792726516724}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:51:52,691 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d: Writing region info on filesystem at 1733237512670Initializing all the Stores at 1733237512671 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237512672 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237512673 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237512673Cleaning up temporary data from old regions at 1733237512684 (+11 ms)Region opened successfully at 1733237512691 (+7 ms) 2024-12-03T14:51:52,691 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b9b4e0a5e7cb9efb1cb8880f73ac6e4d, disabling compactions & flushes 2024-12-03T14:51:52,691 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:52,691 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:52,691 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. after waiting 0 ms 2024-12-03T14:51:52,691 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:52,692 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:52,692 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d: Waiting for close lock at 1733237512691Disabling compacts and flushes for region at 1733237512691Disabling writes for close at 1733237512691Writing region close event to WAL at 1733237512692 (+1 ms)Closed at 1733237512692 2024-12-03T14:51:52,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741878_1055 (size=93) 2024-12-03T14:51:52,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741878_1055 (size=93) 2024-12-03T14:51:52,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741878_1055 (size=93) 2024-12-03T14:51:52,703 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:52,703 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-00545329:(num 1733237512626) 2024-12-03T14:51:52,703 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:52,706 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:52,725 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706, exclude list is [], retry=0 2024-12-03T14:51:52,729 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:52,730 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:52,730 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:52,732 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 2024-12-03T14:51:52,732 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:51:52,733 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b9b4e0a5e7cb9efb1cb8880f73ac6e4d, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:52,733 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:52,733 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,733 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,735 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,736 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName a 2024-12-03T14:51:52,737 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:52,737 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:52,738 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,739 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName b 2024-12-03T14:51:52,739 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:52,740 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:52,740 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,741 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName c 2024-12-03T14:51:52,741 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:52,742 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:52,743 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,743 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,746 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,746 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,746 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:51:52,748 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:52,749 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9b4e0a5e7cb9efb1cb8880f73ac6e4d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68315103, jitterRate=0.017974361777305603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:51:52,750 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d: Writing region info on filesystem at 1733237512733Initializing all the Stores at 1733237512734 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237512734Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237512735 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237512735Cleaning up temporary data from old regions at 1733237512746 (+11 ms)Region opened successfully at 1733237512750 (+4 ms) 2024-12-03T14:51:52,755 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733237512754/Put/seqid=0 2024-12-03T14:51:52,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741881_1058 (size=4826) 2024-12-03T14:51:52,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741881_1058 (size=4826) 2024-12-03T14:51:52,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741881_1058 (size=4826) 2024-12-03T14:51:53,164 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a 2024-12-03T14:51:53,175 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-03T14:51:53,176 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T14:51:53,176 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d: 2024-12-03T14:51:53,178 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a/a6fd6c56203f48eb9a882681d3c339d8_SeqId_3_ 2024-12-03T14:51:53,179 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a as hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a/a6fd6c56203f48eb9a882681d3c339d8_SeqId_3_ - updating store file list. 2024-12-03T14:51:53,185 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a6fd6c56203f48eb9a882681d3c339d8_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T14:51:53,185 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a/a6fd6c56203f48eb9a882681d3c339d8_SeqId_3_ into b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a 2024-12-03T14:51:53,185 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38155/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a (new location: hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a/a6fd6c56203f48eb9a882681d3c339d8_SeqId_3_) 2024-12-03T14:51:53,223 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706, size=0 (0bytes) 2024-12-03T14:51:53,223 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 might be still open, length is 0 2024-12-03T14:51:53,223 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 2024-12-03T14:51:53,223 WARN [IPC Server handler 1 on default port 38155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-03T14:51:53,224 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 after 1ms 2024-12-03T14:51:55,287 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:48034 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37353:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48034 dst: /127.0.0.1:37353 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:37353 remote=/127.0.0.1:48034]. Total timeout mills is 60000, 57905 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:55,288 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:49484 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38931:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49484 dst: /127.0.0.1:38931 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:55,288 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:51258 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51258 dst: /127.0.0.1:40191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:55,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741880_1059 (size=434) 2024-12-03T14:51:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741880_1059 (size=434) 2024-12-03T14:51:55,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741880_1059 (size=434) 2024-12-03T14:51:57,224 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 after 4001ms 2024-12-03T14:51:57,227 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:57,228 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 took 4005ms 2024-12-03T14:51:57,230 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706; continuing. 2024-12-03T14:51:57,230 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 so closing down 2024-12-03T14:51:57,230 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:51:57,232 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733237512706.temp 2024-12-03T14:51:57,234 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005-wal.1733237512706.temp 2024-12-03T14:51:57,234 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:51:57,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741882_1060 (size=236) 2024-12-03T14:51:57,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741882_1060 (size=236) 2024-12-03T14:51:57,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741882_1060 (size=236) 2024-12-03T14:51:57,243 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005-wal.1733237512706.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T14:51:57,244 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005-wal.1733237512706.temp to hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005 2024-12-03T14:51:57,245 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T14:51:57,245 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706, journal: Splitting hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706, size=0 (0bytes) at 1733237513223Finishing writing output for hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 so closing down at 1733237517230 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005-wal.1733237512706.temp at 1733237517234 (+4 ms)3 split writer threads finished at 1733237517234Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005-wal.1733237512706.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733237517243 (+9 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005-wal.1733237512706.temp to hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005 at 1733237517244 (+1 ms)Processed 2 edits across 1 Regions in 17 ms; skipped=1; WAL=hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706, size=0, length=0, corrupted=false, cancelled=false at 1733237517245 (+1 ms) 2024-12-03T14:51:57,246 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237512706 2024-12-03T14:51:57,247 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005 2024-12-03T14:51:57,247 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:51:57,249 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:51:57,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237517249, exclude list is [], retry=0 2024-12-03T14:51:57,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:51:57,263 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:51:57,264 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:51:57,266 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237517249 2024-12-03T14:51:57,287 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:51:57,287 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b9b4e0a5e7cb9efb1cb8880f73ac6e4d, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:57,287 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:57,287 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,288 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,289 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,291 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName a 2024-12-03T14:51:57,291 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:57,300 DEBUG [StoreFileOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a6fd6c56203f48eb9a882681d3c339d8_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T14:51:57,300 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a/a6fd6c56203f48eb9a882681d3c339d8_SeqId_3_ 2024-12-03T14:51:57,300 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:57,300 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,301 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName b 2024-12-03T14:51:57,301 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:57,302 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:57,302 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,303 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9b4e0a5e7cb9efb1cb8880f73ac6e4d columnFamilyName c 2024-12-03T14:51:57,303 DEBUG [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:57,304 INFO [StoreOpener-b9b4e0a5e7cb9efb1cb8880f73ac6e4d-1 {}] regionserver.HStore(327): Store=b9b4e0a5e7cb9efb1cb8880f73ac6e4d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:57,304 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,305 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,306 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,307 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005 2024-12-03T14:51:57,310 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:51:57,311 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005 2024-12-03T14:51:57,311 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b9b4e0a5e7cb9efb1cb8880f73ac6e4d 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-03T14:51:57,330 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/.tmp/a/b71d72f217ff427bafcd3572e401a7c4 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733237513191/Put/seqid=0 2024-12-03T14:51:57,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741884_1062 (size=5149) 2024-12-03T14:51:57,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741884_1062 (size=5149) 2024-12-03T14:51:57,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741884_1062 (size=5149) 2024-12-03T14:51:57,337 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/.tmp/a/b71d72f217ff427bafcd3572e401a7c4 2024-12-03T14:51:57,342 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/.tmp/a/b71d72f217ff427bafcd3572e401a7c4 as hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a/b71d72f217ff427bafcd3572e401a7c4 2024-12-03T14:51:57,347 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a/b71d72f217ff427bafcd3572e401a7c4, entries=1, sequenceid=5, filesize=5.0 K 2024-12-03T14:51:57,347 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b9b4e0a5e7cb9efb1cb8880f73ac6e4d in 36ms, sequenceid=5, compaction requested=false; wal=null 2024-12-03T14:51:57,348 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/0000000000000000005 2024-12-03T14:51:57,349 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,349 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,350 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:51:57,351 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b9b4e0a5e7cb9efb1cb8880f73ac6e4d 2024-12-03T14:51:57,353 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b9b4e0a5e7cb9efb1cb8880f73ac6e4d/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-03T14:51:57,355 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b9b4e0a5e7cb9efb1cb8880f73ac6e4d; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71683973, jitterRate=0.06817443668842316}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:51:57,355 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d: Writing region info on filesystem at 1733237517288Initializing all the Stores at 1733237517289 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517289Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517289Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517289Obtaining lock to block concurrent updates at 1733237517311 (+22 ms)Preparing flush snapshotting stores in b9b4e0a5e7cb9efb1cb8880f73ac6e4d at 1733237517311Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733237517311Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. at 1733237517311Flushing b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a: creating writer at 1733237517311Flushing b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a: appending metadata at 1733237517329 (+18 ms)Flushing b9b4e0a5e7cb9efb1cb8880f73ac6e4d/a: closing flushed file at 1733237517329Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60c12f09: reopening flushed file at 1733237517341 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b9b4e0a5e7cb9efb1cb8880f73ac6e4d in 36ms, sequenceid=5, compaction requested=false; wal=null at 1733237517347 (+6 ms)Cleaning up temporary data from old regions at 1733237517349 (+2 ms)Region opened successfully at 1733237517355 (+6 ms) 2024-12-03T14:51:57,359 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b9b4e0a5e7cb9efb1cb8880f73ac6e4d, disabling compactions & flushes 2024-12-03T14:51:57,359 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:57,359 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:57,360 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. after waiting 0 ms 2024-12-03T14:51:57,360 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:57,361 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733237512596.b9b4e0a5e7cb9efb1cb8880f73ac6e4d. 2024-12-03T14:51:57,361 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b9b4e0a5e7cb9efb1cb8880f73ac6e4d: Waiting for close lock at 1733237517359Disabling compacts and flushes for region at 1733237517359Disabling writes for close at 1733237517360 (+1 ms)Writing region close event to WAL at 1733237517361 (+1 ms)Closed at 1733237517361 2024-12-03T14:51:57,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741883_1061 (size=93) 2024-12-03T14:51:57,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741883_1061 (size=93) 2024-12-03T14:51:57,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741883_1061 (size=93) 2024-12-03T14:51:57,365 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237517249 not finished, retry = 0 2024-12-03T14:51:57,469 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:51:57,469 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733237517249) 2024-12-03T14:51:57,483 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1661450401_22 at /127.0.0.1:51276 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1661450401_22 at /127.0.0.1:49514 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:38155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1661450401_22 at /127.0.0.1:48074 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:38155 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 149), ProcessCount=11 (was 11), AvailableMemoryMB=4679 (was 4797) 2024-12-03T14:51:57,494 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=4677 2024-12-03T14:51:57,509 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:51:57,514 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T14:51:57,518 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:57,521 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2dffa7dd 2024-12-03T14:51:57,522 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T14:51:57,525 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T14:51:57,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T14:51:57,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T14:51:57,541 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T14:51:57,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-03T14:51:57,544 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:57,546 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T14:51:57,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:51:57,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741885_1063 (size=694) 2024-12-03T14:51:57,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741885_1063 (size=694) 2024-12-03T14:51:57,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741885_1063 (size=694) 2024-12-03T14:51:57,567 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cfbb889b168bb28ed91fe90346363f81, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918 2024-12-03T14:51:57,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741886_1064 (size=77) 2024-12-03T14:51:57,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741886_1064 (size=77) 2024-12-03T14:51:57,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741886_1064 (size=77) 2024-12-03T14:51:57,591 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:57,592 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing cfbb889b168bb28ed91fe90346363f81, disabling compactions & flushes 2024-12-03T14:51:57,592 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:57,592 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:57,592 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. after waiting 0 ms 2024-12-03T14:51:57,592 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:57,592 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:57,592 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for cfbb889b168bb28ed91fe90346363f81: Waiting for close lock at 1733237517592Disabling compacts and flushes for region at 1733237517592Disabling writes for close at 1733237517592Writing region close event to WAL at 1733237517592Closed at 1733237517592 2024-12-03T14:51:57,594 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T14:51:57,599 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733237517594"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733237517594"}]},"ts":"1733237517594"} 2024-12-03T14:51:57,602 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T14:51:57,604 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T14:51:57,606 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237517604"}]},"ts":"1733237517604"} 2024-12-03T14:51:57,612 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-03T14:51:57,613 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {a5d22df9eca2=0} racks are {/default-rack=0} 2024-12-03T14:51:57,615 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T14:51:57,615 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T14:51:57,615 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T14:51:57,615 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T14:51:57,616 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T14:51:57,616 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T14:51:57,616 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T14:51:57,616 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T14:51:57,616 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T14:51:57,616 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T14:51:57,618 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN}] 2024-12-03T14:51:57,623 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN 2024-12-03T14:51:57,625 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN; state=OFFLINE, location=a5d22df9eca2,43019,1733237501901; forceNewPlan=false, retain=false 2024-12-03T14:51:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:51:57,778 INFO [a5d22df9eca2:46647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T14:51:57,779 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPENING, regionLocation=a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:57,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN because future has completed 2024-12-03T14:51:57,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901}] 2024-12-03T14:51:57,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:51:57,938 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T14:51:57,940 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32827, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T14:51:57,946 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:57,946 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cfbb889b168bb28ed91fe90346363f81, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:57,947 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,947 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:57,947 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,947 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,949 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,951 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf1 2024-12-03T14:51:57,951 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:57,952 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:57,952 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,953 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf2 2024-12-03T14:51:57,954 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:57,954 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:57,954 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,955 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,956 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,957 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,957 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,958 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T14:51:57,960 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,964 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:51:57,965 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cfbb889b168bb28ed91fe90346363f81; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72054189, jitterRate=0.07369108498096466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T14:51:57,965 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:57,966 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cfbb889b168bb28ed91fe90346363f81: Running coprocessor pre-open hook at 1733237517947Writing region info on filesystem at 1733237517947Initializing all the Stores at 1733237517949 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517949Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237517949Cleaning up temporary data from old regions at 1733237517957 (+8 ms)Running coprocessor post-open hooks at 1733237517965 (+8 ms)Region opened successfully at 1733237517966 (+1 ms) 2024-12-03T14:51:57,968 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., pid=6, masterSystemTime=1733237517938 2024-12-03T14:51:57,973 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:57,973 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:57,974 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPEN, openSeqNum=2, regionLocation=a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:57,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901 because future has completed 2024-12-03T14:51:57,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T14:51:57,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901 in 196 msec 2024-12-03T14:51:57,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T14:51:57,987 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN in 366 msec 2024-12-03T14:51:57,988 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T14:51:57,988 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733237517988"}]},"ts":"1733237517988"} 2024-12-03T14:51:57,991 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-03T14:51:57,992 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T14:51:57,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 461 msec 2024-12-03T14:51:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T14:51:58,175 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-03T14:51:58,175 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-03T14:51:58,176 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:51:58,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-03T14:51:58,183 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T14:51:58,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-03T14:51:58,196 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=2] 2024-12-03T14:51:58,197 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:51:58,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:51:58,215 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=cfbb889b168bb28ed91fe90346363f81, source=a5d22df9eca2,43019,1733237501901, destination=a5d22df9eca2,43671,1733237501988, warming up region on a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:58,217 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=cfbb889b168bb28ed91fe90346363f81, source=a5d22df9eca2,43019,1733237501901, destination=a5d22df9eca2,43671,1733237501988, running balancer 2024-12-03T14:51:58,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE 2024-12-03T14:51:58,219 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE 2024-12-03T14:51:58,221 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=CLOSING, regionLocation=a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:58,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(7855): Warmup {ENCODED => cfbb889b168bb28ed91fe90346363f81, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:58,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:58,223 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE because future has completed 2024-12-03T14:51:58,224 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf1 2024-12-03T14:51:58,224 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:58,225 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:51:58,225 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:58,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901}] 2024-12-03T14:51:58,225 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,226 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf2 2024-12-03T14:51:58,226 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:58,227 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(1722): Closing cfbb889b168bb28ed91fe90346363f81, disabling compactions & flushes 2024-12-03T14:51:58,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. after waiting 0 ms 2024-12-03T14:51:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] regionserver.HRegion(1676): Region close journal for cfbb889b168bb28ed91fe90346363f81: Waiting for close lock at 1733237518227Disabling compacts and flushes for region at 1733237518227Disabling writes for close at 1733237518227Writing region close event to WAL at 1733237518227Closed at 1733237518227 2024-12-03T14:51:58,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-03T14:51:58,383 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,383 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:51:58,384 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing cfbb889b168bb28ed91fe90346363f81, disabling compactions & flushes 2024-12-03T14:51:58,384 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,384 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,384 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. after waiting 0 ms 2024-12-03T14:51:58,384 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,384 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing cfbb889b168bb28ed91fe90346363f81 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-03T14:51:58,405 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf1/2a92c12a9a5d437aba868f75a364c39c is 35, key is r1/cf1:q/1733237518200/Put/seqid=0 2024-12-03T14:51:58,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741887_1065 (size=4783) 2024-12-03T14:51:58,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741887_1065 (size=4783) 2024-12-03T14:51:58,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741887_1065 (size=4783) 2024-12-03T14:51:58,424 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf1/2a92c12a9a5d437aba868f75a364c39c 2024-12-03T14:51:58,435 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf1/2a92c12a9a5d437aba868f75a364c39c as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c 2024-12-03T14:51:58,444 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c, entries=1, sequenceid=5, filesize=4.7 K 2024-12-03T14:51:58,446 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for cfbb889b168bb28ed91fe90346363f81 in 62ms, sequenceid=5, compaction requested=false 2024-12-03T14:51:58,446 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-03T14:51:58,452 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T14:51:58,455 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,456 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for cfbb889b168bb28ed91fe90346363f81: Waiting for close lock at 1733237518384Running coprocessor pre-close hooks at 1733237518384Disabling compacts and flushes for region at 1733237518384Disabling writes for close at 1733237518384Obtaining lock to block concurrent updates at 1733237518384Preparing flush snapshotting stores in cfbb889b168bb28ed91fe90346363f81 at 1733237518384Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733237518385 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. at 1733237518385Flushing cfbb889b168bb28ed91fe90346363f81/cf1: creating writer at 1733237518386 (+1 ms)Flushing cfbb889b168bb28ed91fe90346363f81/cf1: appending metadata at 1733237518404 (+18 ms)Flushing cfbb889b168bb28ed91fe90346363f81/cf1: closing flushed file at 1733237518405 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@332d329: reopening flushed file at 1733237518434 (+29 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for cfbb889b168bb28ed91fe90346363f81 in 62ms, sequenceid=5, compaction requested=false at 1733237518446 (+12 ms)Writing region close event to WAL at 1733237518447 (+1 ms)Running coprocessor post-close hooks at 1733237518453 (+6 ms)Closed at 1733237518455 (+2 ms) 2024-12-03T14:51:58,456 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding cfbb889b168bb28ed91fe90346363f81 move to a5d22df9eca2,43671,1733237501988 record at close sequenceid=5 2024-12-03T14:51:58,459 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,460 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=CLOSED 2024-12-03T14:51:58,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901 because future has completed 2024-12-03T14:51:58,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T14:51:58,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901 in 241 msec 2024-12-03T14:51:58,470 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE; state=CLOSED, location=a5d22df9eca2,43671,1733237501988; forceNewPlan=false, retain=false 2024-12-03T14:51:58,620 INFO [a5d22df9eca2:46647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T14:51:58,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPENING, regionLocation=a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:58,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE because future has completed 2024-12-03T14:51:58,624 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43671,1733237501988}] 2024-12-03T14:51:58,781 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,781 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => cfbb889b168bb28ed91fe90346363f81, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:58,782 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,782 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:58,782 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,782 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,784 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,785 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf1 2024-12-03T14:51:58,785 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:58,794 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c 2024-12-03T14:51:58,794 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:58,795 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,796 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf2 2024-12-03T14:51:58,796 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:58,797 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:58,797 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,798 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,799 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,800 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,800 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,801 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T14:51:58,803 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,803 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened cfbb889b168bb28ed91fe90346363f81; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61525754, jitterRate=-0.08319482207298279}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T14:51:58,803 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:58,804 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for cfbb889b168bb28ed91fe90346363f81: Running coprocessor pre-open hook at 1733237518782Writing region info on filesystem at 1733237518782Initializing all the Stores at 1733237518783 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237518784 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237518784Cleaning up temporary data from old regions at 1733237518800 (+16 ms)Running coprocessor post-open hooks at 1733237518803 (+3 ms)Region opened successfully at 1733237518804 (+1 ms) 2024-12-03T14:51:58,805 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., pid=9, masterSystemTime=1733237518777 2024-12-03T14:51:58,808 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,808 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:58,808 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPEN, openSeqNum=9, regionLocation=a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:58,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43671,1733237501988 because future has completed 2024-12-03T14:51:58,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-03T14:51:58,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43671,1733237501988 in 188 msec 2024-12-03T14:51:58,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE in 597 msec 2024-12-03T14:51:58,824 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T14:51:58,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T14:51:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:43070 deadline: 1733237578830, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43671 startCode=1733237501988. As of locationSeqNum=5. 2024-12-03T14:51:58,853 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43671 startCode=1733237501988. As of locationSeqNum=5. 2024-12-03T14:51:58,853 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43671 startCode=1733237501988. As of locationSeqNum=5. 2024-12-03T14:51:58,853 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43671,1733237501988, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43671 startCode=1733237501988. As of locationSeqNum=5. 2024-12-03T14:51:58,974 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cfbb889b168bb28ed91fe90346363f81 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-03T14:51:58,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf1/aab47735d4ce4b35b0c312f2cd39a941 is 29, key is r1/cf1:/1733237518965/DeleteFamily/seqid=0 2024-12-03T14:51:58,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741888_1066 (size=4906) 2024-12-03T14:51:58,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741888_1066 (size=4906) 2024-12-03T14:51:58,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741888_1066 (size=4906) 2024-12-03T14:51:58,998 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf1/aab47735d4ce4b35b0c312f2cd39a941 2024-12-03T14:51:59,004 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aab47735d4ce4b35b0c312f2cd39a941 2024-12-03T14:51:59,017 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf2/8dea417af6764c25b47f0ed528423f0f is 29, key is r1/cf2:/1733237518965/DeleteFamily/seqid=0 2024-12-03T14:51:59,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741889_1067 (size=4906) 2024-12-03T14:51:59,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741889_1067 (size=4906) 2024-12-03T14:51:59,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741889_1067 (size=4906) 2024-12-03T14:51:59,023 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf2/8dea417af6764c25b47f0ed528423f0f 2024-12-03T14:51:59,028 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8dea417af6764c25b47f0ed528423f0f 2024-12-03T14:51:59,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf1/aab47735d4ce4b35b0c312f2cd39a941 as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/aab47735d4ce4b35b0c312f2cd39a941 2024-12-03T14:51:59,035 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aab47735d4ce4b35b0c312f2cd39a941 2024-12-03T14:51:59,035 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/aab47735d4ce4b35b0c312f2cd39a941, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T14:51:59,037 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf2/8dea417af6764c25b47f0ed528423f0f as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/8dea417af6764c25b47f0ed528423f0f 2024-12-03T14:51:59,044 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8dea417af6764c25b47f0ed528423f0f 2024-12-03T14:51:59,044 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/8dea417af6764c25b47f0ed528423f0f, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T14:51:59,045 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for cfbb889b168bb28ed91fe90346363f81 in 71ms, sequenceid=12, compaction requested=false 2024-12-03T14:51:59,045 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cfbb889b168bb28ed91fe90346363f81: 2024-12-03T14:51:59,048 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T14:51:59,049 DEBUG [Time-limited test {}] regionserver.HStore(1541): cfbb889b168bb28ed91fe90346363f81/cf1 is initiating major compaction (all files) 2024-12-03T14:51:59,049 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T14:51:59,050 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:59,050 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of cfbb889b168bb28ed91fe90346363f81/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,050 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c, hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/aab47735d4ce4b35b0c312f2cd39a941] into tmpdir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp, totalSize=9.5 K 2024-12-03T14:51:59,051 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2a92c12a9a5d437aba868f75a364c39c, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733237518200 2024-12-03T14:51:59,052 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting aab47735d4ce4b35b0c312f2cd39a941, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T14:51:59,063 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): cfbb889b168bb28ed91fe90346363f81#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:51:59,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741890_1068 (size=4626) 2024-12-03T14:51:59,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741890_1068 (size=4626) 2024-12-03T14:51:59,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741890_1068 (size=4626) 2024-12-03T14:51:59,078 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf1/b4047641c5144e9094902024a5020ad5 as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/b4047641c5144e9094902024a5020ad5 2024-12-03T14:51:59,091 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in cfbb889b168bb28ed91fe90346363f81/cf1 of cfbb889b168bb28ed91fe90346363f81 into b4047641c5144e9094902024a5020ad5(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:51:59,091 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for cfbb889b168bb28ed91fe90346363f81: 2024-12-03T14:51:59,092 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T14:51:59,092 DEBUG [Time-limited test {}] regionserver.HStore(1541): cfbb889b168bb28ed91fe90346363f81/cf2 is initiating major compaction (all files) 2024-12-03T14:51:59,092 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T14:51:59,092 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T14:51:59,092 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of cfbb889b168bb28ed91fe90346363f81/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,092 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/8dea417af6764c25b47f0ed528423f0f] into tmpdir=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp, totalSize=4.8 K 2024-12-03T14:51:59,093 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 8dea417af6764c25b47f0ed528423f0f, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T14:51:59,099 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): cfbb889b168bb28ed91fe90346363f81#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T14:51:59,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741891_1069 (size=4592) 2024-12-03T14:51:59,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741891_1069 (size=4592) 2024-12-03T14:51:59,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741891_1069 (size=4592) 2024-12-03T14:51:59,114 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/.tmp/cf2/f8d26cf010b14ae2bd514835df73ac6c as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/f8d26cf010b14ae2bd514835df73ac6c 2024-12-03T14:51:59,120 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in cfbb889b168bb28ed91fe90346363f81/cf2 of cfbb889b168bb28ed91fe90346363f81 into f8d26cf010b14ae2bd514835df73ac6c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:51:59,121 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for cfbb889b168bb28ed91fe90346363f81: 2024-12-03T14:51:59,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=cfbb889b168bb28ed91fe90346363f81, source=a5d22df9eca2,43671,1733237501988, destination=a5d22df9eca2,43019,1733237501901, warming up region on a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:59,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=cfbb889b168bb28ed91fe90346363f81, source=a5d22df9eca2,43671,1733237501988, destination=a5d22df9eca2,43019,1733237501901, running balancer 2024-12-03T14:51:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE 2024-12-03T14:51:59,127 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE 2024-12-03T14:51:59,129 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=CLOSING, regionLocation=a5d22df9eca2,43671,1733237501988 2024-12-03T14:51:59,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(7855): Warmup {ENCODED => cfbb889b168bb28ed91fe90346363f81, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:59,130 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,131 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf1 2024-12-03T14:51:59,131 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:59,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE because future has completed 2024-12-03T14:51:59,132 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T14:51:59,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43671,1733237501988}] 2024-12-03T14:51:59,141 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c 2024-12-03T14:51:59,146 INFO [StoreFileOpener-cfbb889b168bb28ed91fe90346363f81-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for aab47735d4ce4b35b0c312f2cd39a941 2024-12-03T14:51:59,146 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/aab47735d4ce4b35b0c312f2cd39a941 2024-12-03T14:51:59,152 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/b4047641c5144e9094902024a5020ad5 2024-12-03T14:51:59,152 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:59,152 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,153 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf2 2024-12-03T14:51:59,153 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:59,161 INFO [StoreFileOpener-cfbb889b168bb28ed91fe90346363f81-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8dea417af6764c25b47f0ed528423f0f 2024-12-03T14:51:59,161 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/8dea417af6764c25b47f0ed528423f0f 2024-12-03T14:51:59,166 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/f8d26cf010b14ae2bd514835df73ac6c 2024-12-03T14:51:59,167 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(1722): Closing cfbb889b168bb28ed91fe90346363f81, disabling compactions & flushes 2024-12-03T14:51:59,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. after waiting 0 ms 2024-12-03T14:51:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43019 {}] regionserver.HRegion(1676): Region close journal for cfbb889b168bb28ed91fe90346363f81: Waiting for close lock at 1733237519167Disabling compacts and flushes for region at 1733237519167Disabling writes for close at 1733237519167Writing region close event to WAL at 1733237519169 (+2 ms)Closed at 1733237519169 2024-12-03T14:51:59,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-03T14:51:59,287 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,287 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T14:51:59,287 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing cfbb889b168bb28ed91fe90346363f81, disabling compactions & flushes 2024-12-03T14:51:59,287 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,287 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,288 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. after waiting 0 ms 2024-12-03T14:51:59,288 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,288 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c, hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/aab47735d4ce4b35b0c312f2cd39a941] to archive 2024-12-03T14:51:59,293 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:51:59,297 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c to hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/2a92c12a9a5d437aba868f75a364c39c 2024-12-03T14:51:59,299 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/aab47735d4ce4b35b0c312f2cd39a941 to hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/aab47735d4ce4b35b0c312f2cd39a941 2024-12-03T14:51:59,312 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/8dea417af6764c25b47f0ed528423f0f] to archive 2024-12-03T14:51:59,313 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:51:59,315 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/8dea417af6764c25b47f0ed528423f0f to hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/8dea417af6764c25b47f0ed528423f0f 2024-12-03T14:51:59,321 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-03T14:51:59,321 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,321 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for cfbb889b168bb28ed91fe90346363f81: Waiting for close lock at 1733237519287Running coprocessor pre-close hooks at 1733237519287Disabling compacts and flushes for region at 1733237519287Disabling writes for close at 1733237519288 (+1 ms)Writing region close event to WAL at 1733237519317 (+29 ms)Running coprocessor post-close hooks at 1733237519321 (+4 ms)Closed at 1733237519321 2024-12-03T14:51:59,322 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding cfbb889b168bb28ed91fe90346363f81 move to a5d22df9eca2,43019,1733237501901 record at close sequenceid=12 2024-12-03T14:51:59,324 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,325 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=CLOSED 2024-12-03T14:51:59,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43671,1733237501988 because future has completed 2024-12-03T14:51:59,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T14:51:59,332 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43671,1733237501988 in 197 msec 2024-12-03T14:51:59,332 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE; state=CLOSED, location=a5d22df9eca2,43019,1733237501901; forceNewPlan=false, retain=false 2024-12-03T14:51:59,483 INFO [a5d22df9eca2:46647 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T14:51:59,483 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPENING, regionLocation=a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:59,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE because future has completed 2024-12-03T14:51:59,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901}] 2024-12-03T14:51:59,649 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,649 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => cfbb889b168bb28ed91fe90346363f81, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:51:59,650 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,650 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:51:59,650 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,650 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,653 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,654 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf1 2024-12-03T14:51:59,654 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:59,664 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/b4047641c5144e9094902024a5020ad5 2024-12-03T14:51:59,664 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:59,664 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,665 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf2 2024-12-03T14:51:59,666 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:51:59,673 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/f8d26cf010b14ae2bd514835df73ac6c 2024-12-03T14:51:59,674 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:51:59,674 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,675 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,676 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,677 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,677 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,677 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T14:51:59,679 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,679 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened cfbb889b168bb28ed91fe90346363f81; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66058661, jitterRate=-0.01564924418926239}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T14:51:59,679 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,680 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for cfbb889b168bb28ed91fe90346363f81: Running coprocessor pre-open hook at 1733237519650Writing region info on filesystem at 1733237519650Initializing all the Stores at 1733237519652 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237519652Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237519652Cleaning up temporary data from old regions at 1733237519677 (+25 ms)Running coprocessor post-open hooks at 1733237519680 (+3 ms)Region opened successfully at 1733237519680 2024-12-03T14:51:59,681 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., pid=12, masterSystemTime=1733237519644 2024-12-03T14:51:59,683 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,683 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPEN, openSeqNum=18, regionLocation=a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:59,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901 because future has completed 2024-12-03T14:51:59,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-03T14:51:59,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,43019,1733237501901 in 197 msec 2024-12-03T14:51:59,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, REOPEN/MOVE in 564 msec 2024-12-03T14:51:59,730 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T14:51:59,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T14:51:59,736 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server a5d22df9eca2,43019,1733237501901: testing ***** 2024-12-03T14:51:59,737 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T14:51:59,740 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T14:51:59,742 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T14:51:59,749 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T14:51:59,750 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T14:51:59,759 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 273133056 }, "NonHeapMemoryUsage": { "committed": 171704320, "init": 7667712, "max": -1, "used": 168989536 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "a5d22df9eca2", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 43, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 43, "ProcessCallTime_99th_percentile": 43, "ProcessCallTime_99.9th_percentile": 43, "ProcessCallTime_TimeRangeCount_0-1": 14, "ProcessCallTime_TimeRangeCount_1-3": 10, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 43, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 6, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 43, "TotalCallTime_99th_percentile": 43, "TotalCallTime_99.9th_percentile": 43, "TotalCallTime_TimeRangeCount_0-1": 13, "TotalCallTime_TimeRangeCount_1-3": 11, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "a5d22df9eca2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:51554", "tag.serverName": "a5d22df9eca2,43671,1733237501988", "tag.clusterId": "1ef03b02-7d7a-4dc2-9ea3-fcb035dd7514", "tag.Context": "regionserver", "tag.Hostname": "a5d22df9eca2", "regionCount": 1, "storeCount": 4, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 1767, "memStoreHeapSize": 3864, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733237501988, "averageRegionSize": 1767, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 1.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 532480, "ByteBuffAllocatorHeapAllocationRatio": 0.400849396798649, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 28, "totalRowActionRequestCount": 14, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 7, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 16, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 28, "MajorCompactionTime_max": 40, "MajorCompactionTime_mean": 34, "MajorCompactionTime_25th_percentile": 28, "MajorCompactionTime_median": 29, "MajorCompactionTime_75th_percentile": 40, "MajorCompactionTime_90th_percentile": 40, "MajorCompactionTime_95th_percentile": 40, "MajorCompactionTime_98th_percentile": 40, "MajorCompactionTime_99th_percentile": 40, "MajorCompactionTime_99.9th_percentile": 40, "MajorCompactionTime_TimeRangeCount_10-30": 1, "MajorCompactionTime_TimeRangeCount_30-100": 1, "ScanTime_num_ops": 9, "ScanTime_min": 1, "ScanTime_max": 1, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 5, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 15, "Put_min": 1, "Put_max": 3, "Put_mean": 1, "Put_25th_percentile": 1, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 2, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_0-1": 6, "Put_TimeRangeCount_1-3": 7, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 2, "Delete_max": 2, "Delete_mean": 2, "Delete_25th_percentile": 2, "Delete_median": 2, "Delete_75th_percentile": 2, "Delete_90th_percentile": 2, "Delete_95th_percentile": 2, "Delete_98th_percentile": 2, "Delete_99th_percentile": 2, "Delete_99.9th_percentile": 2, "Delete_TimeRangeCount_1-3": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 28, "CompactionTime_max": 40, "CompactionTime_mean": 34, "CompactionTime_25th_percentile": 28, "CompactionTime_median": 29, "CompactionTime_75th_percentile": 40, "CompactionTime_90th_percentile": 40, "CompactionTime_95th_percentile": 40, "CompactionTime_98th_percentile": 40, "CompactionTime_99th_percentile": 40, "CompactionTime_99.9th_percentile": 40, "CompactionTime_TimeRangeCount_10-30": 1, "CompactionTime_TimeRangeCount_30-100": 1, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 71, "FlushTime_max": 71, "FlushTime_mean": 71, "FlushTime_25th_percentile": 71, "FlushTime_median": 71, "FlushTime_75th_percentile": 71, "FlushTime_90th_percentile": 71, "FlushTime_95th_percentile": 71, "FlushTime_98th_percentile": 71, "FlushTime_99th_percentile": 71, "FlushTime_99.9th_percentile": 71, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 1112, "ScanSize_mean": 889, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_0-10": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T14:51:59,762 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46647 {}] master.MasterRpcServices(700): a5d22df9eca2,43019,1733237501901 reported a fatal error: ***** ABORTING region server a5d22df9eca2,43019,1733237501901: testing ***** 2024-12-03T14:51:59,764 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,43019,1733237501901' ***** 2024-12-03T14:51:59,764 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-03T14:51:59,764 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T14:51:59,765 INFO [RS:0;a5d22df9eca2:43019 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T14:51:59,765 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T14:51:59,765 INFO [RS:0;a5d22df9eca2:43019 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T14:51:59,765 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(3091): Received CLOSE for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43671 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:48012 deadline: 1733237579765, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43019 startCode=1733237501901. As of locationSeqNum=12. 2024-12-03T14:51:59,765 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(956): aborting server a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:59,765 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:51:59,766 INFO [RS:0;a5d22df9eca2:43019 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a5d22df9eca2:43019. 2024-12-03T14:51:59,766 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cfbb889b168bb28ed91fe90346363f81, disabling compactions & flushes 2024-12-03T14:51:59,766 DEBUG [RS:0;a5d22df9eca2:43019 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:51:59,766 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,766 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43671,1733237501988, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43671,1733237501988, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43019 startCode=1733237501901. As of locationSeqNum=12. 2024-12-03T14:51:59,766 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,766 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43671,1733237501988, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43019 startCode=1733237501901. As of locationSeqNum=12. 2024-12-03T14:51:59,766 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. after waiting 0 ms 2024-12-03T14:51:59,766 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43671,1733237501988, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=a5d22df9eca2 port=43019 startCode=1733237501901. As of locationSeqNum=12. 2024-12-03T14:51:59,766 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,766 DEBUG [RS:0;a5d22df9eca2:43019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:51:59,766 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T14:51:59,767 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1325): Online Regions={cfbb889b168bb28ed91fe90346363f81=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.} 2024-12-03T14:51:59,767 DEBUG [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1351): Waiting on cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:51:59,769 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,770 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cfbb889b168bb28ed91fe90346363f81: Waiting for close lock at 1733237519765Running coprocessor pre-close hooks at 1733237519766 (+1 ms)Disabling compacts and flushes for region at 1733237519766Disabling writes for close at 1733237519766Writing region close event to WAL at 1733237519769 (+3 ms)Running coprocessor post-close hooks at 1733237519769Closed at 1733237519769 2024-12-03T14:51:59,770 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:51:59,789 INFO [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:51:59,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a5d22df9eca2,43019,1733237501901 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:51:59,877 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a5d22df9eca2,43019,1733237501901 aborting 2024-12-03T14:51:59,877 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server a5d22df9eca2,43019,1733237501901 aborting 2024-12-03T14:51:59,877 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=12 from cache 2024-12-03T14:51:59,967 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,43019,1733237501901; all regions closed. 2024-12-03T14:51:59,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741835_1011 (size=1677) 2024-12-03T14:51:59,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741835_1011 (size=1677) 2024-12-03T14:51:59,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741835_1011 (size=1677) 2024-12-03T14:51:59,978 DEBUG [RS:0;a5d22df9eca2:43019 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:51:59,979 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:51:59,979 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:51:59,979 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T14:51:59,980 INFO [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:51:59,980 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T14:51:59,980 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T14:51:59,980 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T14:51:59,980 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:51:59,981 INFO [RS:0;a5d22df9eca2:43019 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43019 2024-12-03T14:51:59,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,43019,1733237501901 2024-12-03T14:51:59,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:51:59,986 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:51:59,988 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,43019,1733237501901] 2024-12-03T14:51:59,989 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,43019,1733237501901 already deleted, retry=false 2024-12-03T14:51:59,989 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a5d22df9eca2,43019,1733237501901 on a5d22df9eca2,46647,1733237501335 2024-12-03T14:51:59,993 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a5d22df9eca2,43019,1733237501901, splitWal=true, meta=false 2024-12-03T14:51:59,996 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a5d22df9eca2,43019,1733237501901, splitWal=true, meta=false 2024-12-03T14:51:59,997 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for a5d22df9eca2,43019,1733237501901 (carryingMeta=false) a5d22df9eca2,43019,1733237501901/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@5af9a216[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T14:51:59,998 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): a5d22df9eca2,43019,1733237501901 had 1 regions 2024-12-03T14:51:59,999 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure a5d22df9eca2,43019,1733237501901, splitWal=true, meta=false, isMeta: false 2024-12-03T14:52:00,001 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting 2024-12-03T14:52:00,002 INFO [PEWorker-1 {}] master.SplitWALManager(105): a5d22df9eca2,43019,1733237501901 WAL count=1, meta=false 2024-12-03T14:52:00,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure a5d22df9eca2%2C43019%2C1733237501901.1733237503225}] 2024-12-03T14:52:00,010 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=a5d22df9eca2,43671,1733237501988 2024-12-03T14:52:00,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure a5d22df9eca2%2C43019%2C1733237501901.1733237503225, worker=a5d22df9eca2,43671,1733237501988}] 2024-12-03T14:52:00,087 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18] 2024-12-03T14:52:00,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:00,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43019-0x100a090c71d0001, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:00,089 INFO [RS:0;a5d22df9eca2:43019 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:52:00,089 INFO [RS:0;a5d22df9eca2:43019 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,43019,1733237501901; zookeeper connection closed. 2024-12-03T14:52:00,089 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server a5d22df9eca2:43019 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a5d22df9eca2/172.17.0.2:43019 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:00,089 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@480587a4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@480587a4 2024-12-03T14:52:00,089 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18, error=java.net.ConnectException: Call to address=a5d22df9eca2:43019 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a5d22df9eca2/172.17.0.2:43019 2024-12-03T14:52:00,090 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-03T14:52:00,090 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18 from cache 2024-12-03T14:52:00,090 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address a5d22df9eca2:43019 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: a5d22df9eca2/172.17.0.2:43019 2024-12-03T14:52:00,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=43671 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-03T14:52:00,188 INFO [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225, size=1.6 K (1677bytes) 2024-12-03T14:52:00,189 INFO [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 2024-12-03T14:52:00,189 INFO [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 after 0ms 2024-12-03T14:52:00,192 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:00,192 INFO [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 took 4ms 2024-12-03T14:52:00,198 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for cfbb889b168bb28ed91fe90346363f81: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-03T14:52:00,199 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 so closing down 2024-12-03T14:52:00,199 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:52:00,199 INFO [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:52:00,199 INFO [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-12-03T14:52:00,199 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225, journal: Splitting hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225, size=1.6 K (1677bytes) at 1733237520188Finishing writing output for hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 so closing down at 1733237520199 (+11 ms)3 split writer threads finished at 1733237520199Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1733237520199 2024-12-03T14:52:00,199 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 2024-12-03T14:52:00,201 DEBUG [RS_LOG_REPLAY_OPS-regionserver/a5d22df9eca2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-03T14:52:00,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46647 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-03T14:52:00,208 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting/a5d22df9eca2%2C43019%2C1733237501901.1733237503225 to hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs 2024-12-03T14:52:00,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T14:52:00,211 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure a5d22df9eca2%2C43019%2C1733237501901.1733237503225, worker=a5d22df9eca2,43671,1733237501988 in 196 msec 2024-12-03T14:52:00,213 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=a5d22df9eca2,43671,1733237501988 2024-12-03T14:52:00,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T14:52:00,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure a5d22df9eca2%2C43019%2C1733237501901.1733237503225, worker=a5d22df9eca2,43671,1733237501988 in 210 msec 2024-12-03T14:52:00,219 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting dir is empty, no logs to split. 2024-12-03T14:52:00,219 INFO [PEWorker-1 {}] master.SplitWALManager(105): a5d22df9eca2,43019,1733237501901 WAL count=0, meta=false 2024-12-03T14:52:00,219 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if a5d22df9eca2,43019,1733237501901 WAL splitting is done? wals=0, meta=false 2024-12-03T14:52:00,221 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for a5d22df9eca2,43019,1733237501901 failed, ignore...File hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43019,1733237501901-splitting does not exist. 2024-12-03T14:52:00,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN}] 2024-12-03T14:52:00,224 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN 2024-12-03T14:52:00,225 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T14:52:00,376 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(204): Hosts are {a5d22df9eca2=0} racks are {/default-rack=0} 2024-12-03T14:52:00,377 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T14:52:00,377 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T14:52:00,377 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T14:52:00,377 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T14:52:00,377 INFO [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T14:52:00,377 INFO [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T14:52:00,378 DEBUG [a5d22df9eca2:46647 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T14:52:00,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPENING, regionLocation=a5d22df9eca2,38259,1733237502024 2024-12-03T14:52:00,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN because future has completed 2024-12-03T14:52:00,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,38259,1733237502024}] 2024-12-03T14:52:00,396 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18] 2024-12-03T14:52:00,397 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to a5d22df9eca2:43019 this server is in the failed servers list 2024-12-03T14:52:00,398 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=a5d22df9eca2:43019 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a5d22df9eca2:43019 2024-12-03T14:52:00,398 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: a5d22df9eca2:43019 2024-12-03T14:52:00,398 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,43019,1733237501901, seqNum=18 from cache 2024-12-03T14:52:00,539 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T14:52:00,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52947, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T14:52:00,547 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:00,547 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => cfbb889b168bb28ed91fe90346363f81, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:00,548 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,548 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:00,548 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,549 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,551 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,553 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf1 2024-12-03T14:52:00,553 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:00,562 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf1/b4047641c5144e9094902024a5020ad5 2024-12-03T14:52:00,563 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:00,563 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,564 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfbb889b168bb28ed91fe90346363f81 columnFamilyName cf2 2024-12-03T14:52:00,564 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:00,570 DEBUG [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/cf2/f8d26cf010b14ae2bd514835df73ac6c 2024-12-03T14:52:00,570 INFO [StoreOpener-cfbb889b168bb28ed91fe90346363f81-1 {}] regionserver.HStore(327): Store=cfbb889b168bb28ed91fe90346363f81/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:00,570 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,571 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,572 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,573 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,573 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,574 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T14:52:00,575 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,576 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened cfbb889b168bb28ed91fe90346363f81; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61417290, jitterRate=-0.08481106162071228}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T14:52:00,576 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:00,577 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for cfbb889b168bb28ed91fe90346363f81: Running coprocessor pre-open hook at 1733237520549Writing region info on filesystem at 1733237520549Initializing all the Stores at 1733237520550 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237520551 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237520551Cleaning up temporary data from old regions at 1733237520573 (+22 ms)Running coprocessor post-open hooks at 1733237520576 (+3 ms)Region opened successfully at 1733237520577 (+1 ms) 2024-12-03T14:52:00,578 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., pid=17, masterSystemTime=1733237520538 2024-12-03T14:52:00,580 DEBUG [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:00,580 INFO [RS_OPEN_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:00,580 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=cfbb889b168bb28ed91fe90346363f81, regionState=OPEN, openSeqNum=18, regionLocation=a5d22df9eca2,38259,1733237502024 2024-12-03T14:52:00,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,38259,1733237502024 because future has completed 2024-12-03T14:52:00,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T14:52:00,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure cfbb889b168bb28ed91fe90346363f81, server=a5d22df9eca2,38259,1733237502024 in 199 msec 2024-12-03T14:52:00,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-03T14:52:00,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=cfbb889b168bb28ed91fe90346363f81, ASSIGN in 364 msec 2024-12-03T14:52:00,589 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server a5d22df9eca2,43019,1733237501901 after splitting done 2024-12-03T14:52:00,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure a5d22df9eca2,43019,1733237501901, splitWal=true, meta=false in 599 msec 2024-12-03T14:52:00,909 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81., hostname=a5d22df9eca2,38259,1733237502024, seqNum=18] 2024-12-03T14:52:00,909 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T14:52:00,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T14:52:00,925 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=408 (was 411), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=4613 (was 4677) 2024-12-03T14:52:00,927 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-03T14:52:00,937 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=408, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=4613 2024-12-03T14:52:00,937 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-12-03T14:52:00,950 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:00,952 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:00,952 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:52:00,955 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-09195362, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-09195362, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:00,966 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-09195362/hregion-09195362.1733237520955, exclude list is [], retry=0 2024-12-03T14:52:00,969 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:00,969 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:00,970 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:00,973 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-09195362/hregion-09195362.1733237520955 2024-12-03T14:52:00,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:52:00,974 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => bbcff667ecbdf3b0f63963f37f25b791, NAME => 'testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:52:00,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741893_1071 (size=67) 2024-12-03T14:52:00,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741893_1071 (size=67) 2024-12-03T14:52:00,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741893_1071 (size=67) 2024-12-03T14:52:00,982 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:00,983 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,984 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName a 2024-12-03T14:52:00,984 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:00,985 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:00,985 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,986 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName b 2024-12-03T14:52:00,986 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:00,987 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:00,987 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,988 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName c 2024-12-03T14:52:00,988 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:00,988 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:00,988 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,989 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,989 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,990 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,990 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,991 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:00,992 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:00,994 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:52:00,994 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bbcff667ecbdf3b0f63963f37f25b791; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65733771, jitterRate=-0.020490482449531555}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:00,995 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bbcff667ecbdf3b0f63963f37f25b791: Writing region info on filesystem at 1733237520982Initializing all the Stores at 1733237520983 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237520983Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237520983Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237520983Cleaning up temporary data from old regions at 1733237520990 (+7 ms)Region opened successfully at 1733237520995 (+5 ms) 2024-12-03T14:52:00,995 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bbcff667ecbdf3b0f63963f37f25b791, disabling compactions & flushes 2024-12-03T14:52:00,995 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:00,995 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:00,995 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. after waiting 0 ms 2024-12-03T14:52:00,995 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:00,995 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:00,996 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bbcff667ecbdf3b0f63963f37f25b791: Waiting for close lock at 1733237520995Disabling compacts and flushes for region at 1733237520995Disabling writes for close at 1733237520995Writing region close event to WAL at 1733237520995Closed at 1733237520995 2024-12-03T14:52:00,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741892_1070 (size=93) 2024-12-03T14:52:00,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741892_1070 (size=93) 2024-12-03T14:52:00,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741892_1070 (size=93) 2024-12-03T14:52:01,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:52:01,001 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-09195362:(num 1733237520955) 2024-12-03T14:52:01,001 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:01,003 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:01,014 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003, exclude list is [], retry=0 2024-12-03T14:52:01,017 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:01,017 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:01,017 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:01,018 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003 2024-12-03T14:52:01,019 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:52:01,019 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bbcff667ecbdf3b0f63963f37f25b791, NAME => 'testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:01,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:01,019 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,019 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,021 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,021 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName a 2024-12-03T14:52:01,022 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,022 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,022 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,023 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName b 2024-12-03T14:52:01,023 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,023 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,024 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,024 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName c 2024-12-03T14:52:01,024 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,025 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,025 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,027 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,027 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,028 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:01,029 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,030 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bbcff667ecbdf3b0f63963f37f25b791; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61909653, jitterRate=-0.07747428119182587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:01,030 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bbcff667ecbdf3b0f63963f37f25b791: Writing region info on filesystem at 1733237521019Initializing all the Stores at 1733237521020 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521020Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521020Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521020Cleaning up temporary data from old regions at 1733237521027 (+7 ms)Region opened successfully at 1733237521030 (+3 ms) 2024-12-03T14:52:01,060 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bbcff667ecbdf3b0f63963f37f25b791 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T14:52:01,078 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/a/f5b67f7601c54d478fa0c8832f9c708b is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733237521030/Put/seqid=0 2024-12-03T14:52:01,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741895_1073 (size=5958) 2024-12-03T14:52:01,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741895_1073 (size=5958) 2024-12-03T14:52:01,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741895_1073 (size=5958) 2024-12-03T14:52:01,085 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/a/f5b67f7601c54d478fa0c8832f9c708b 2024-12-03T14:52:01,104 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/b/59a77b7f49704b53b7896df819f7f39b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733237521038/Put/seqid=0 2024-12-03T14:52:01,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741896_1074 (size=5958) 2024-12-03T14:52:01,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741896_1074 (size=5958) 2024-12-03T14:52:01,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741896_1074 (size=5958) 2024-12-03T14:52:01,114 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/b/59a77b7f49704b53b7896df819f7f39b 2024-12-03T14:52:01,140 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/c/cd4bda1dcd294dd9b346a94d5de6a998 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733237521047/Put/seqid=0 2024-12-03T14:52:01,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741897_1075 (size=5958) 2024-12-03T14:52:01,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741897_1075 (size=5958) 2024-12-03T14:52:01,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741897_1075 (size=5958) 2024-12-03T14:52:01,148 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/c/cd4bda1dcd294dd9b346a94d5de6a998 2024-12-03T14:52:01,153 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/a/f5b67f7601c54d478fa0c8832f9c708b as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/a/f5b67f7601c54d478fa0c8832f9c708b 2024-12-03T14:52:01,160 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/a/f5b67f7601c54d478fa0c8832f9c708b, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T14:52:01,161 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/b/59a77b7f49704b53b7896df819f7f39b as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/b/59a77b7f49704b53b7896df819f7f39b 2024-12-03T14:52:01,168 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/b/59a77b7f49704b53b7896df819f7f39b, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T14:52:01,169 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/c/cd4bda1dcd294dd9b346a94d5de6a998 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/c/cd4bda1dcd294dd9b346a94d5de6a998 2024-12-03T14:52:01,178 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/c/cd4bda1dcd294dd9b346a94d5de6a998, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T14:52:01,180 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for bbcff667ecbdf3b0f63963f37f25b791 in 120ms, sequenceid=33, compaction requested=false 2024-12-03T14:52:01,180 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bbcff667ecbdf3b0f63963f37f25b791: 2024-12-03T14:52:01,180 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing bbcff667ecbdf3b0f63963f37f25b791, disabling compactions & flushes 2024-12-03T14:52:01,180 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:01,180 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:01,180 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. after waiting 0 ms 2024-12-03T14:52:01,180 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:01,182 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. 2024-12-03T14:52:01,182 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for bbcff667ecbdf3b0f63963f37f25b791: Waiting for close lock at 1733237521180Disabling compacts and flushes for region at 1733237521180Disabling writes for close at 1733237521180Writing region close event to WAL at 1733237521182 (+2 ms)Closed at 1733237521182 2024-12-03T14:52:01,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741894_1072 (size=2873) 2024-12-03T14:52:01,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741894_1072 (size=2873) 2024-12-03T14:52:01,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741894_1072 (size=2873) 2024-12-03T14:52:01,191 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/b/59a77b7f49704b53b7896df819f7f39b to hdfs://localhost:38155/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/b/59a77b7f49704b53b7896df819f7f39b 2024-12-03T14:52:01,206 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003, size=2.8 K (2873bytes) 2024-12-03T14:52:01,207 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003 2024-12-03T14:52:01,207 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003 after 0ms 2024-12-03T14:52:01,209 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:01,210 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003 took 3ms 2024-12-03T14:52:01,212 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003 so closing down 2024-12-03T14:52:01,212 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:52:01,213 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733237521003.temp 2024-12-03T14:52:01,214 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000003-wal.1733237521003.temp 2024-12-03T14:52:01,215 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:52:01,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741898_1076 (size=2312) 2024-12-03T14:52:01,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741898_1076 (size=2312) 2024-12-03T14:52:01,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741898_1076 (size=2312) 2024-12-03T14:52:01,223 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000003-wal.1733237521003.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T14:52:01,224 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000003-wal.1733237521003.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000032 2024-12-03T14:52:01,224 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-12-03T14:52:01,224 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003, journal: Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003, size=2.8 K (2873bytes) at 1733237521207Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003 so closing down at 1733237521212 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000003-wal.1733237521003.temp at 1733237521215 (+3 ms)3 split writer threads finished at 1733237521215Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000003-wal.1733237521003.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733237521223 (+8 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000003-wal.1733237521003.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000032 at 1733237521224 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1733237521224 2024-12-03T14:52:01,227 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521003 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237521003 2024-12-03T14:52:01,228 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000032 2024-12-03T14:52:01,228 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:01,230 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:01,246 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521231, exclude list is [], retry=0 2024-12-03T14:52:01,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:01,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:01,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:01,251 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521231 2024-12-03T14:52:01,251 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:52:01,251 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => bbcff667ecbdf3b0f63963f37f25b791, NAME => 'testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:01,251 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:01,251 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,251 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,253 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,254 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName a 2024-12-03T14:52:01,254 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,260 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/a/f5b67f7601c54d478fa0c8832f9c708b 2024-12-03T14:52:01,260 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,260 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,262 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName b 2024-12-03T14:52:01,262 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,263 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,263 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,264 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbcff667ecbdf3b0f63963f37f25b791 columnFamilyName c 2024-12-03T14:52:01,264 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,270 DEBUG [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/c/cd4bda1dcd294dd9b346a94d5de6a998 2024-12-03T14:52:01,271 INFO [StoreOpener-bbcff667ecbdf3b0f63963f37f25b791-1 {}] regionserver.HStore(327): Store=bbcff667ecbdf3b0f63963f37f25b791/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,271 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,273 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,274 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000032 2024-12-03T14:52:01,277 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:01,278 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000032 2024-12-03T14:52:01,279 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bbcff667ecbdf3b0f63963f37f25b791 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T14:52:01,297 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/b/09149551cd3c4a57a4202f45343d044c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733237521038/Put/seqid=0 2024-12-03T14:52:01,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741900_1078 (size=5958) 2024-12-03T14:52:01,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741900_1078 (size=5958) 2024-12-03T14:52:01,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741900_1078 (size=5958) 2024-12-03T14:52:01,305 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/b/09149551cd3c4a57a4202f45343d044c 2024-12-03T14:52:01,312 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/.tmp/b/09149551cd3c4a57a4202f45343d044c as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/b/09149551cd3c4a57a4202f45343d044c 2024-12-03T14:52:01,318 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/b/09149551cd3c4a57a4202f45343d044c, entries=10, sequenceid=32, filesize=5.8 K 2024-12-03T14:52:01,319 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for bbcff667ecbdf3b0f63963f37f25b791 in 39ms, sequenceid=32, compaction requested=false; wal=null 2024-12-03T14:52:01,319 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/0000000000000000032 2024-12-03T14:52:01,320 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,320 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,321 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:01,322 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for bbcff667ecbdf3b0f63963f37f25b791 2024-12-03T14:52:01,324 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/bbcff667ecbdf3b0f63963f37f25b791/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-03T14:52:01,325 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened bbcff667ecbdf3b0f63963f37f25b791; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75397271, jitterRate=0.12350688874721527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:01,326 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for bbcff667ecbdf3b0f63963f37f25b791: Writing region info on filesystem at 1733237521251Initializing all the Stores at 1733237521252 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521252Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521252Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521252Obtaining lock to block concurrent updates at 1733237521279 (+27 ms)Preparing flush snapshotting stores in bbcff667ecbdf3b0f63963f37f25b791 at 1733237521279Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733237521279Flushing stores of testReplayEditsWrittenViaHRegion,,1733237520950.bbcff667ecbdf3b0f63963f37f25b791. at 1733237521279Flushing bbcff667ecbdf3b0f63963f37f25b791/b: creating writer at 1733237521279Flushing bbcff667ecbdf3b0f63963f37f25b791/b: appending metadata at 1733237521296 (+17 ms)Flushing bbcff667ecbdf3b0f63963f37f25b791/b: closing flushed file at 1733237521296Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61397742: reopening flushed file at 1733237521311 (+15 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for bbcff667ecbdf3b0f63963f37f25b791 in 39ms, sequenceid=32, compaction requested=false; wal=null at 1733237521319 (+8 ms)Cleaning up temporary data from old regions at 1733237521320 (+1 ms)Region opened successfully at 1733237521326 (+6 ms) 2024-12-03T14:52:01,353 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=415 (was 408) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:51276 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:42680 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:49514 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:48074 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:48528 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:43420 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1125 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 137), ProcessCount=11 (was 11), AvailableMemoryMB=4604 (was 4613) 2024-12-03T14:52:01,354 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-12-03T14:52:01,368 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1125, MaxFileDescriptor=1048576, SystemLoadAverage=137, ProcessCount=11, AvailableMemoryMB=4606 2024-12-03T14:52:01,368 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-12-03T14:52:01,387 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:01,389 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:01,390 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:52:01,392 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15420644, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-15420644, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:01,408 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15420644/hregion-15420644.1733237521393, exclude list is [], retry=0 2024-12-03T14:52:01,411 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:01,411 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:01,411 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:01,414 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15420644/hregion-15420644.1733237521393 2024-12-03T14:52:01,415 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:52:01,415 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d9e0da25b643ea3b323fc5794ef71460, NAME => 'testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:52:01,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741902_1080 (size=68) 2024-12-03T14:52:01,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741902_1080 (size=68) 2024-12-03T14:52:01,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741902_1080 (size=68) 2024-12-03T14:52:01,423 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:01,424 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,426 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName a 2024-12-03T14:52:01,426 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,427 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,427 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,428 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName b 2024-12-03T14:52:01,429 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,430 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,430 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,432 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName c 2024-12-03T14:52:01,432 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,432 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,433 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,434 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,434 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,435 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,435 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,436 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:01,436 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,438 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:52:01,439 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9e0da25b643ea3b323fc5794ef71460; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71468498, jitterRate=0.06496360898017883}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:01,440 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9e0da25b643ea3b323fc5794ef71460: Writing region info on filesystem at 1733237521423Initializing all the Stores at 1733237521424 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521424Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521424Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521424Cleaning up temporary data from old regions at 1733237521435 (+11 ms)Region opened successfully at 1733237521439 (+4 ms) 2024-12-03T14:52:01,440 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9e0da25b643ea3b323fc5794ef71460, disabling compactions & flushes 2024-12-03T14:52:01,440 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:01,440 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:01,440 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. after waiting 0 ms 2024-12-03T14:52:01,440 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:01,441 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:01,441 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9e0da25b643ea3b323fc5794ef71460: Waiting for close lock at 1733237521440Disabling compacts and flushes for region at 1733237521440Disabling writes for close at 1733237521440Writing region close event to WAL at 1733237521441 (+1 ms)Closed at 1733237521441 2024-12-03T14:52:01,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741901_1079 (size=93) 2024-12-03T14:52:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741901_1079 (size=93) 2024-12-03T14:52:01,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741901_1079 (size=93) 2024-12-03T14:52:01,449 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:52:01,449 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15420644:(num 1733237521393) 2024-12-03T14:52:01,449 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:01,451 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:01,462 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451, exclude list is [], retry=0 2024-12-03T14:52:01,464 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:01,464 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:01,464 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:01,465 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451 2024-12-03T14:52:01,466 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:52:01,527 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d9e0da25b643ea3b323fc5794ef71460, NAME => 'testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:01,529 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,529 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:01,530 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,530 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,532 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,532 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName a 2024-12-03T14:52:01,533 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,533 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,533 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,534 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName b 2024-12-03T14:52:01,534 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,535 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,535 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,536 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName c 2024-12-03T14:52:01,536 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:01,537 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:01,537 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,537 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,539 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,540 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,540 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,540 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:01,542 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,543 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9e0da25b643ea3b323fc5794ef71460; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68768186, jitterRate=0.02472582459449768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:01,543 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:01,544 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9e0da25b643ea3b323fc5794ef71460: Running coprocessor pre-open hook at 1733237521530Writing region info on filesystem at 1733237521530Initializing all the Stores at 1733237521531 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521531Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521531Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237521531Cleaning up temporary data from old regions at 1733237521540 (+9 ms)Running coprocessor post-open hooks at 1733237521543 (+3 ms)Region opened successfully at 1733237521543 2024-12-03T14:52:01,556 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9e0da25b643ea3b323fc5794ef71460 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-03T14:52:01,557 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:01,660 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-03T14:52:01,660 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-03T14:52:01,661 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-03T14:52:01,661 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-03T14:52:01,662 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T14:52:01,662 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-03T14:52:01,663 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-03T14:52:01,663 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-03T14:52:01,664 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-03T14:52:01,664 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-03T14:52:02,558 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:03,477 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:52:03,560 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:04,560 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:05,562 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:06,563 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:07,565 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:08,566 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:09,567 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:10,514 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T14:52:10,568 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for d9e0da25b643ea3b323fc5794ef71460/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:10,570 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d9e0da25b643ea3b323fc5794ef71460: 2024-12-03T14:52:10,570 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:10,592 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d9e0da25b643ea3b323fc5794ef71460: 2024-12-03T14:52:10,593 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-03T14:52:10,593 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9e0da25b643ea3b323fc5794ef71460, disabling compactions & flushes 2024-12-03T14:52:10,593 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:10,593 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:10,593 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. after waiting 0 ms 2024-12-03T14:52:10,593 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:10,594 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:10,594 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. 2024-12-03T14:52:10,594 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9e0da25b643ea3b323fc5794ef71460: Waiting for close lock at 1733237530593Running coprocessor pre-close hooks at 1733237530593Disabling compacts and flushes for region at 1733237530593Disabling writes for close at 1733237530593Writing region close event to WAL at 1733237530594 (+1 ms)Running coprocessor post-close hooks at 1733237530594Closed at 1733237530594 2024-12-03T14:52:10,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741903_1081 (size=2398) 2024-12-03T14:52:10,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741903_1081 (size=2398) 2024-12-03T14:52:10,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741903_1081 (size=2398) 2024-12-03T14:52:10,624 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451, size=2.3 K (2398bytes) 2024-12-03T14:52:10,624 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451 2024-12-03T14:52:10,624 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451 after 0ms 2024-12-03T14:52:10,627 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:10,628 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451 took 5ms 2024-12-03T14:52:10,630 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451 so closing down 2024-12-03T14:52:10,630 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:52:10,631 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733237521451.temp 2024-12-03T14:52:10,633 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000004-wal.1733237521451.temp 2024-12-03T14:52:10,633 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:52:10,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741904_1082 (size=1672) 2024-12-03T14:52:10,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741904_1082 (size=1672) 2024-12-03T14:52:10,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741904_1082 (size=1672) 2024-12-03T14:52:10,645 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000004-wal.1733237521451.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-03T14:52:10,646 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000004-wal.1733237521451.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000026 2024-12-03T14:52:10,647 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-12-03T14:52:10,647 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451, journal: Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451, size=2.3 K (2398bytes) at 1733237530624Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451 so closing down at 1733237530630 (+6 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000004-wal.1733237521451.temp at 1733237530633 (+3 ms)3 split writer threads finished at 1733237530633Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000004-wal.1733237521451.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733237530645 (+12 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000004-wal.1733237521451.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000026 at 1733237530646 (+1 ms)Processed 23 edits across 1 Regions in 18 ms; skipped=3; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1733237530647 (+1 ms) 2024-12-03T14:52:10,649 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237521451 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237521451 2024-12-03T14:52:10,650 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000026 2024-12-03T14:52:10,650 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:10,652 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:10,668 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237530653, exclude list is [], retry=0 2024-12-03T14:52:10,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:10,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:10,672 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:10,674 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237530653 2024-12-03T14:52:10,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:52:10,675 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d9e0da25b643ea3b323fc5794ef71460, NAME => 'testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:10,676 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,676 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:10,676 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,676 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,678 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,679 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName a 2024-12-03T14:52:10,679 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:10,680 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:10,680 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,681 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName b 2024-12-03T14:52:10,681 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:10,682 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:10,682 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,683 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9e0da25b643ea3b323fc5794ef71460 columnFamilyName c 2024-12-03T14:52:10,683 DEBUG [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:10,684 INFO [StoreOpener-d9e0da25b643ea3b323fc5794ef71460-1 {}] regionserver.HStore(327): Store=d9e0da25b643ea3b323fc5794ef71460/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:10,684 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,685 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,686 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,687 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000026 2024-12-03T14:52:10,689 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:10,691 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000026 2024-12-03T14:52:10,691 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9e0da25b643ea3b323fc5794ef71460 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-03T14:52:10,706 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/a/f507abc1981e482297bc668fe9698b9e is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733237530577/Put/seqid=0 2024-12-03T14:52:10,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741906_1084 (size=5523) 2024-12-03T14:52:10,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741906_1084 (size=5523) 2024-12-03T14:52:10,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741906_1084 (size=5523) 2024-12-03T14:52:10,714 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/a/f507abc1981e482297bc668fe9698b9e 2024-12-03T14:52:10,733 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/b/3f07cc6f5e414d60a431cb6415bbeb62 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733237530571/Put/seqid=0 2024-12-03T14:52:10,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741907_1085 (size=5524) 2024-12-03T14:52:10,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741907_1085 (size=5524) 2024-12-03T14:52:10,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741907_1085 (size=5524) 2024-12-03T14:52:10,740 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/b/3f07cc6f5e414d60a431cb6415bbeb62 2024-12-03T14:52:10,759 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/c/bad538689a364f398cb8cf151ba8dc3e is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733237530574/Put/seqid=0 2024-12-03T14:52:10,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741908_1086 (size=5457) 2024-12-03T14:52:10,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741908_1086 (size=5457) 2024-12-03T14:52:10,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741908_1086 (size=5457) 2024-12-03T14:52:10,766 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/c/bad538689a364f398cb8cf151ba8dc3e 2024-12-03T14:52:10,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/a/f507abc1981e482297bc668fe9698b9e as hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/a/f507abc1981e482297bc668fe9698b9e 2024-12-03T14:52:10,778 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/a/f507abc1981e482297bc668fe9698b9e, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T14:52:10,779 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/b/3f07cc6f5e414d60a431cb6415bbeb62 as hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/b/3f07cc6f5e414d60a431cb6415bbeb62 2024-12-03T14:52:10,785 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/b/3f07cc6f5e414d60a431cb6415bbeb62, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T14:52:10,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/.tmp/c/bad538689a364f398cb8cf151ba8dc3e as hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/c/bad538689a364f398cb8cf151ba8dc3e 2024-12-03T14:52:10,792 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/c/bad538689a364f398cb8cf151ba8dc3e, entries=6, sequenceid=26, filesize=5.3 K 2024-12-03T14:52:10,792 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d9e0da25b643ea3b323fc5794ef71460 in 101ms, sequenceid=26, compaction requested=false; wal=null 2024-12-03T14:52:10,793 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/0000000000000000026 2024-12-03T14:52:10,794 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,794 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,795 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:10,796 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,799 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsAfterAbortingFlush/d9e0da25b643ea3b323fc5794ef71460/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-03T14:52:10,800 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9e0da25b643ea3b323fc5794ef71460; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62435977, jitterRate=-0.06963144242763519}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:10,800 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d9e0da25b643ea3b323fc5794ef71460 2024-12-03T14:52:10,801 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9e0da25b643ea3b323fc5794ef71460: Running coprocessor pre-open hook at 1733237530676Writing region info on filesystem at 1733237530676Initializing all the Stores at 1733237530677 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237530677Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237530677Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237530677Obtaining lock to block concurrent updates at 1733237530691 (+14 ms)Preparing flush snapshotting stores in d9e0da25b643ea3b323fc5794ef71460 at 1733237530691Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733237530692 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733237521387.d9e0da25b643ea3b323fc5794ef71460. at 1733237530692Flushing d9e0da25b643ea3b323fc5794ef71460/a: creating writer at 1733237530692Flushing d9e0da25b643ea3b323fc5794ef71460/a: appending metadata at 1733237530705 (+13 ms)Flushing d9e0da25b643ea3b323fc5794ef71460/a: closing flushed file at 1733237530705Flushing d9e0da25b643ea3b323fc5794ef71460/b: creating writer at 1733237530719 (+14 ms)Flushing d9e0da25b643ea3b323fc5794ef71460/b: appending metadata at 1733237530733 (+14 ms)Flushing d9e0da25b643ea3b323fc5794ef71460/b: closing flushed file at 1733237530733Flushing d9e0da25b643ea3b323fc5794ef71460/c: creating writer at 1733237530746 (+13 ms)Flushing d9e0da25b643ea3b323fc5794ef71460/c: appending metadata at 1733237530759 (+13 ms)Flushing d9e0da25b643ea3b323fc5794ef71460/c: closing flushed file at 1733237530759Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c9fe580: reopening flushed file at 1733237530771 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21958677: reopening flushed file at 1733237530778 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@237556db: reopening flushed file at 1733237530785 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for d9e0da25b643ea3b323fc5794ef71460 in 101ms, sequenceid=26, compaction requested=false; wal=null at 1733237530792 (+7 ms)Cleaning up temporary data from old regions at 1733237530794 (+2 ms)Running coprocessor post-open hooks at 1733237530800 (+6 ms)Region opened successfully at 1733237530801 (+1 ms) 2024-12-03T14:52:10,822 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=414 (was 415), OpenFileDescriptor=1182 (was 1125) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 137) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4596 (was 4606) 2024-12-03T14:52:10,822 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-03T14:52:10,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=414, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=4596 2024-12-03T14:52:10,833 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-03T14:52:10,845 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:10,846 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:10,847 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:52:10,849 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-11928017, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-11928017, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:10,859 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-11928017/hregion-11928017.1733237530849, exclude list is [], retry=0 2024-12-03T14:52:10,862 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:10,862 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:10,862 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:10,864 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-11928017/hregion-11928017.1733237530849 2024-12-03T14:52:10,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:52:10,865 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b3c037f15b4387d911c85e5f63f973e2, NAME => 'testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:52:10,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741910_1088 (size=61) 2024-12-03T14:52:10,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741910_1088 (size=61) 2024-12-03T14:52:10,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741910_1088 (size=61) 2024-12-03T14:52:10,874 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:10,875 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,876 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3c037f15b4387d911c85e5f63f973e2 columnFamilyName a 2024-12-03T14:52:10,876 DEBUG [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:10,877 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(327): Store=b3c037f15b4387d911c85e5f63f973e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:10,877 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,878 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,878 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,880 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,882 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:52:10,882 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b3c037f15b4387d911c85e5f63f973e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69431988, jitterRate=0.03461724519729614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:52:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b3c037f15b4387d911c85e5f63f973e2: Writing region info on filesystem at 1733237530874Initializing all the Stores at 1733237530875 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237530875Cleaning up temporary data from old regions at 1733237530878 (+3 ms)Region opened successfully at 1733237530883 (+5 ms) 2024-12-03T14:52:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b3c037f15b4387d911c85e5f63f973e2, disabling compactions & flushes 2024-12-03T14:52:10,883 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. after waiting 0 ms 2024-12-03T14:52:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,884 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,884 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b3c037f15b4387d911c85e5f63f973e2: Waiting for close lock at 1733237530883Disabling compacts and flushes for region at 1733237530883Disabling writes for close at 1733237530883Writing region close event to WAL at 1733237530883Closed at 1733237530883 2024-12-03T14:52:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741909_1087 (size=93) 2024-12-03T14:52:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741909_1087 (size=93) 2024-12-03T14:52:10,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741909_1087 (size=93) 2024-12-03T14:52:10,888 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:52:10,888 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-11928017:(num 1733237530849) 2024-12-03T14:52:10,888 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:10,890 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:10,900 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890, exclude list is [], retry=0 2024-12-03T14:52:10,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:10,903 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:10,903 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:10,904 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890 2024-12-03T14:52:10,905 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:52:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b3c037f15b4387d911c85e5f63f973e2, NAME => 'testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,905 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,908 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,909 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3c037f15b4387d911c85e5f63f973e2 columnFamilyName a 2024-12-03T14:52:10,909 DEBUG [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:10,909 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(327): Store=b3c037f15b4387d911c85e5f63f973e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:10,909 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,910 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,911 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,911 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,913 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:10,914 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b3c037f15b4387d911c85e5f63f973e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61969303, jitterRate=-0.07658542692661285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:52:10,914 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b3c037f15b4387d911c85e5f63f973e2: Writing region info on filesystem at 1733237530905Initializing all the Stores at 1733237530906 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237530906Cleaning up temporary data from old regions at 1733237530911 (+5 ms)Region opened successfully at 1733237530914 (+3 ms) 2024-12-03T14:52:10,924 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b3c037f15b4387d911c85e5f63f973e2, disabling compactions & flushes 2024-12-03T14:52:10,924 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,924 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,924 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. after waiting 0 ms 2024-12-03T14:52:10,925 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,925 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,925 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. 2024-12-03T14:52:10,925 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b3c037f15b4387d911c85e5f63f973e2: Waiting for close lock at 1733237530924Disabling compacts and flushes for region at 1733237530924Disabling writes for close at 1733237530925 (+1 ms)Writing region close event to WAL at 1733237530925Closed at 1733237530925 2024-12-03T14:52:10,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741911_1089 (size=838) 2024-12-03T14:52:10,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741911_1089 (size=838) 2024-12-03T14:52:10,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741911_1089 (size=838) 2024-12-03T14:52:10,949 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890, size=838 (838bytes) 2024-12-03T14:52:10,949 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890 2024-12-03T14:52:10,949 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890 after 0ms 2024-12-03T14:52:10,952 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:10,952 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890 took 4ms 2024-12-03T14:52:10,954 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890 so closing down 2024-12-03T14:52:10,954 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:52:10,955 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733237530890.temp 2024-12-03T14:52:10,956 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000003-wal.1733237530890.temp 2024-12-03T14:52:10,956 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:52:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741912_1090 (size=838) 2024-12-03T14:52:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741912_1090 (size=838) 2024-12-03T14:52:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741912_1090 (size=838) 2024-12-03T14:52:10,966 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000003-wal.1733237530890.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-03T14:52:10,967 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000003-wal.1733237530890.temp to hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012 2024-12-03T14:52:10,967 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890, size=838, length=838, corrupted=false, cancelled=false 2024-12-03T14:52:10,967 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890, journal: Splitting hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890, size=838 (838bytes) at 1733237530949Finishing writing output for hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890 so closing down at 1733237530954 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000003-wal.1733237530890.temp at 1733237530956 (+2 ms)3 split writer threads finished at 1733237530956Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000003-wal.1733237530890.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733237530966 (+10 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000003-wal.1733237530890.temp to hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012 at 1733237530967 (+1 ms)Processed 10 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890, size=838, length=838, corrupted=false, cancelled=false at 1733237530967 2024-12-03T14:52:10,969 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237530890 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237530890 2024-12-03T14:52:10,970 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012 2024-12-03T14:52:10,973 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:11,268 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:11,270 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:11,281 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237531271, exclude list is [], retry=0 2024-12-03T14:52:11,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:11,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:11,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:11,285 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237531271 2024-12-03T14:52:11,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:52:11,286 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b3c037f15b4387d911c85e5f63f973e2, NAME => 'testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:11,286 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:11,286 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,286 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,289 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,290 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3c037f15b4387d911c85e5f63f973e2 columnFamilyName a 2024-12-03T14:52:11,291 DEBUG [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,291 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(327): Store=b3c037f15b4387d911c85e5f63f973e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,291 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,292 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,293 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,294 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012 2024-12-03T14:52:11,296 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:11,297 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012 2024-12-03T14:52:11,297 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b3c037f15b4387d911c85e5f63f973e2 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-03T14:52:11,319 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/.tmp/a/e1c1cae7a38d4a4fb446936ebc64f1a1 is 79, key is testDatalossWhenInputError/a:x0/1733237530914/Put/seqid=0 2024-12-03T14:52:11,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741914_1092 (size=5808) 2024-12-03T14:52:11,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741914_1092 (size=5808) 2024-12-03T14:52:11,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741914_1092 (size=5808) 2024-12-03T14:52:11,327 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/.tmp/a/e1c1cae7a38d4a4fb446936ebc64f1a1 2024-12-03T14:52:11,336 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/.tmp/a/e1c1cae7a38d4a4fb446936ebc64f1a1 as hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/a/e1c1cae7a38d4a4fb446936ebc64f1a1 2024-12-03T14:52:11,345 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/a/e1c1cae7a38d4a4fb446936ebc64f1a1, entries=10, sequenceid=12, filesize=5.7 K 2024-12-03T14:52:11,345 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b3c037f15b4387d911c85e5f63f973e2 in 48ms, sequenceid=12, compaction requested=false; wal=null 2024-12-03T14:52:11,346 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/0000000000000000012 2024-12-03T14:52:11,347 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,347 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,349 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,352 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-03T14:52:11,352 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b3c037f15b4387d911c85e5f63f973e2; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62412578, jitterRate=-0.06998011469841003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:52:11,353 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b3c037f15b4387d911c85e5f63f973e2: Writing region info on filesystem at 1733237531286Initializing all the Stores at 1733237531289 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531289Obtaining lock to block concurrent updates at 1733237531297 (+8 ms)Preparing flush snapshotting stores in b3c037f15b4387d911c85e5f63f973e2 at 1733237531297Finished memstore snapshotting testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733237531297Flushing stores of testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2. at 1733237531297Flushing b3c037f15b4387d911c85e5f63f973e2/a: creating writer at 1733237531297Flushing b3c037f15b4387d911c85e5f63f973e2/a: appending metadata at 1733237531318 (+21 ms)Flushing b3c037f15b4387d911c85e5f63f973e2/a: closing flushed file at 1733237531318Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77057215: reopening flushed file at 1733237531334 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for b3c037f15b4387d911c85e5f63f973e2 in 48ms, sequenceid=12, compaction requested=false; wal=null at 1733237531345 (+11 ms)Cleaning up temporary data from old regions at 1733237531347 (+2 ms)Region opened successfully at 1733237531353 (+6 ms) 2024-12-03T14:52:11,357 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b3c037f15b4387d911c85e5f63f973e2, NAME => 'testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:11,357 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733237530845.b3c037f15b4387d911c85e5f63f973e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:11,357 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,357 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,358 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,359 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3c037f15b4387d911c85e5f63f973e2 columnFamilyName a 2024-12-03T14:52:11,359 DEBUG [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,364 DEBUG [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/a/e1c1cae7a38d4a4fb446936ebc64f1a1 2024-12-03T14:52:11,364 INFO [StoreOpener-b3c037f15b4387d911c85e5f63f973e2-1 {}] regionserver.HStore(327): Store=b3c037f15b4387d911c85e5f63f973e2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,364 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,365 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,367 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,367 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,367 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,369 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b3c037f15b4387d911c85e5f63f973e2 2024-12-03T14:52:11,371 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testDatalossWhenInputError/b3c037f15b4387d911c85e5f63f973e2/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-03T14:52:11,373 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b3c037f15b4387d911c85e5f63f973e2; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72602931, jitterRate=0.08186797797679901}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T14:52:11,373 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b3c037f15b4387d911c85e5f63f973e2: Writing region info on filesystem at 1733237531357Initializing all the Stores at 1733237531358 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531358Cleaning up temporary data from old regions at 1733237531367 (+9 ms)Region opened successfully at 1733237531373 (+6 ms) 2024-12-03T14:52:11,390 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=424 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:35708 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:35646 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:52602 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:42390 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:42466 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-870883219-172.17.0.2-1733237498670:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:52492 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=4588 (was 4596) 2024-12-03T14:52:11,390 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-03T14:52:11,400 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=4587 2024-12-03T14:52:11,401 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-03T14:52:11,415 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:11,416 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:11,417 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:52:11,419 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67477244, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-67477244, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:11,431 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67477244/hregion-67477244.1733237531420, exclude list is [], retry=0 2024-12-03T14:52:11,433 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:11,433 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:11,434 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:11,436 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67477244/hregion-67477244.1733237531420 2024-12-03T14:52:11,436 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:52:11,436 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d0de749bcd5f79c66a37c8be66963f1a, NAME => 'testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:52:11,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741916_1094 (size=63) 2024-12-03T14:52:11,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741916_1094 (size=63) 2024-12-03T14:52:11,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741916_1094 (size=63) 2024-12-03T14:52:11,449 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:11,451 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,452 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName a 2024-12-03T14:52:11,452 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,452 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,453 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,454 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName b 2024-12-03T14:52:11,454 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,454 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,455 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,456 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName c 2024-12-03T14:52:11,456 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,456 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,457 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,457 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,458 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,459 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,459 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,459 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:11,460 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,462 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:52:11,462 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0de749bcd5f79c66a37c8be66963f1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63637494, jitterRate=-0.05172744393348694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:11,463 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0de749bcd5f79c66a37c8be66963f1a: Writing region info on filesystem at 1733237531449Initializing all the Stores at 1733237531450 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531450Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531450Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531450Cleaning up temporary data from old regions at 1733237531459 (+9 ms)Region opened successfully at 1733237531463 (+4 ms) 2024-12-03T14:52:11,463 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0de749bcd5f79c66a37c8be66963f1a, disabling compactions & flushes 2024-12-03T14:52:11,463 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:11,463 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:11,463 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. after waiting 0 ms 2024-12-03T14:52:11,463 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:11,463 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:11,464 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0de749bcd5f79c66a37c8be66963f1a: Waiting for close lock at 1733237531463Disabling compacts and flushes for region at 1733237531463Disabling writes for close at 1733237531463Writing region close event to WAL at 1733237531463Closed at 1733237531463 2024-12-03T14:52:11,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741915_1093 (size=93) 2024-12-03T14:52:11,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741915_1093 (size=93) 2024-12-03T14:52:11,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741915_1093 (size=93) 2024-12-03T14:52:11,467 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:52:11,468 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67477244:(num 1733237531420) 2024-12-03T14:52:11,468 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:11,469 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:11,480 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470, exclude list is [], retry=0 2024-12-03T14:52:11,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:11,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:11,482 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:11,484 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 2024-12-03T14:52:11,484 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:52:11,484 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0de749bcd5f79c66a37c8be66963f1a, NAME => 'testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:11,484 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:11,484 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,484 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,485 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,486 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName a 2024-12-03T14:52:11,486 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,487 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,487 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,488 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName b 2024-12-03T14:52:11,488 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,489 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,489 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,490 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName c 2024-12-03T14:52:11,490 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:11,490 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:11,490 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,491 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,493 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,493 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,493 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:11,494 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:11,495 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0de749bcd5f79c66a37c8be66963f1a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74612091, jitterRate=0.11180679500102997}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:11,496 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0de749bcd5f79c66a37c8be66963f1a: Writing region info on filesystem at 1733237531484Initializing all the Stores at 1733237531485 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531485Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531485Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237531485Cleaning up temporary data from old regions at 1733237531493 (+8 ms)Region opened successfully at 1733237531496 (+3 ms) 2024-12-03T14:52:11,502 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733237531502/Put/seqid=0 2024-12-03T14:52:11,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741918_1096 (size=4875) 2024-12-03T14:52:11,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741918_1096 (size=4875) 2024-12-03T14:52:11,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741918_1096 (size=4875) 2024-12-03T14:52:11,513 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733237531512/Put/seqid=0 2024-12-03T14:52:11,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741919_1097 (size=4875) 2024-12-03T14:52:11,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741919_1097 (size=4875) 2024-12-03T14:52:11,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741919_1097 (size=4875) 2024-12-03T14:52:11,526 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733237531526/Put/seqid=0 2024-12-03T14:52:11,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741920_1098 (size=4875) 2024-12-03T14:52:11,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741920_1098 (size=4875) 2024-12-03T14:52:11,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741920_1098 (size=4875) 2024-12-03T14:52:11,536 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in d0de749bcd5f79c66a37c8be66963f1a/a 2024-12-03T14:52:11,540 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-03T14:52:11,540 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T14:52:11,540 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in d0de749bcd5f79c66a37c8be66963f1a/a 2024-12-03T14:52:11,543 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-03T14:52:11,543 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T14:52:11,543 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in d0de749bcd5f79c66a37c8be66963f1a/a 2024-12-03T14:52:11,546 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-03T14:52:11,546 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T14:52:11,547 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d0de749bcd5f79c66a37c8be66963f1a 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-03T14:52:11,562 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/.tmp/a/d55895ea68da42ba90183d0f541fc0a8 is 55, key is testCompactedBulkLoadedFiles/a:a/1733237531496/Put/seqid=0 2024-12-03T14:52:11,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741921_1099 (size=5107) 2024-12-03T14:52:11,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741921_1099 (size=5107) 2024-12-03T14:52:11,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741921_1099 (size=5107) 2024-12-03T14:52:11,568 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/.tmp/a/d55895ea68da42ba90183d0f541fc0a8 2024-12-03T14:52:11,573 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/.tmp/a/d55895ea68da42ba90183d0f541fc0a8 as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8 2024-12-03T14:52:11,578 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8, entries=1, sequenceid=4, filesize=5.0 K 2024-12-03T14:52:11,579 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for d0de749bcd5f79c66a37c8be66963f1a in 32ms, sequenceid=4, compaction requested=false 2024-12-03T14:52:11,579 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d0de749bcd5f79c66a37c8be66963f1a: 2024-12-03T14:52:11,580 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_ 2024-12-03T14:52:11,582 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_ 2024-12-03T14:52:11,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_ 2024-12-03T14:52:11,584 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile0 into d0de749bcd5f79c66a37c8be66963f1a/a as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_ - updating store file list. 2024-12-03T14:52:11,589 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for b5f7118471ef4726939b28dd2f0100a9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T14:52:11,590 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_ into d0de749bcd5f79c66a37c8be66963f1a/a 2024-12-03T14:52:11,590 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile0 into d0de749bcd5f79c66a37c8be66963f1a/a (new location: hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_) 2024-12-03T14:52:11,591 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile1 into d0de749bcd5f79c66a37c8be66963f1a/a as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_ - updating store file list. 2024-12-03T14:52:11,596 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T14:52:11,596 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_ into d0de749bcd5f79c66a37c8be66963f1a/a 2024-12-03T14:52:11,596 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile1 into d0de749bcd5f79c66a37c8be66963f1a/a (new location: hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_) 2024-12-03T14:52:11,597 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile2 into d0de749bcd5f79c66a37c8be66963f1a/a as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_ - updating store file list. 2024-12-03T14:52:11,602 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 82f88874c0ca44df972a7826dac791d8_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T14:52:11,602 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_ into d0de749bcd5f79c66a37c8be66963f1a/a 2024-12-03T14:52:11,602 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:38155/hbase/testCompactedBulkLoadedFiles/hfile2 into d0de749bcd5f79c66a37c8be66963f1a/a (new location: hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_) 2024-12-03T14:52:11,609 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T14:52:11,609 DEBUG [Time-limited test {}] regionserver.HStore(1541): d0de749bcd5f79c66a37c8be66963f1a/a is initiating major compaction (all files) 2024-12-03T14:52:11,609 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of d0de749bcd5f79c66a37c8be66963f1a/a in testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:11,609 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_] into tmpdir=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/.tmp, totalSize=19.3 K 2024-12-03T14:52:11,610 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d55895ea68da42ba90183d0f541fc0a8, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733237531496 2024-12-03T14:52:11,610 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b5f7118471ef4726939b28dd2f0100a9_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T14:52:11,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T14:52:11,611 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 82f88874c0ca44df972a7826dac791d8_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T14:52:11,621 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/.tmp/a/f9afe04b97214e4e88911f37b4761dad is 55, key is testCompactedBulkLoadedFiles/a:a/1733237531496/Put/seqid=0 2024-12-03T14:52:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741922_1100 (size=6154) 2024-12-03T14:52:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741922_1100 (size=6154) 2024-12-03T14:52:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741922_1100 (size=6154) 2024-12-03T14:52:11,634 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/.tmp/a/f9afe04b97214e4e88911f37b4761dad as hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/f9afe04b97214e4e88911f37b4761dad 2024-12-03T14:52:11,639 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in d0de749bcd5f79c66a37c8be66963f1a/a of d0de749bcd5f79c66a37c8be66963f1a into f9afe04b97214e4e88911f37b4761dad(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T14:52:11,639 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for d0de749bcd5f79c66a37c8be66963f1a: 2024-12-03T14:52:11,639 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T14:52:11,640 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T14:52:11,660 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-03T14:52:11,660 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-03T14:52:11,661 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-03T14:52:11,661 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-03T14:52:11,670 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470, size=0 (0bytes) 2024-12-03T14:52:11,670 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 might be still open, length is 0 2024-12-03T14:52:11,670 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 2024-12-03T14:52:11,671 WARN [IPC Server handler 1 on default port 38155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-03T14:52:11,671 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 after 1ms 2024-12-03T14:52:13,292 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:42514 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:38931:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42514 dst: /127.0.0.1:38931 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38931 remote=/127.0.0.1:42514]. Total timeout mills is 60000, 58348 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:13,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:52628 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37353:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52628 dst: /127.0.0.1:37353 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:13,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:35738 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35738 dst: /127.0.0.1:40191 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:13,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741917_1101 (size=1368) 2024-12-03T14:52:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741917_1101 (size=1368) 2024-12-03T14:52:13,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741917_1101 (size=1368) 2024-12-03T14:52:15,672 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 after 4002ms 2024-12-03T14:52:15,674 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:15,674 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 took 4004ms 2024-12-03T14:52:15,676 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470; continuing. 2024-12-03T14:52:15,676 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 so closing down 2024-12-03T14:52:15,676 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:52:15,678 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733237531470.temp 2024-12-03T14:52:15,679 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000003-wal.1733237531470.temp 2024-12-03T14:52:15,679 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:52:15,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741923_1102 (size=635) 2024-12-03T14:52:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741923_1102 (size=635) 2024-12-03T14:52:15,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741923_1102 (size=635) 2024-12-03T14:52:15,687 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000003-wal.1733237531470.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T14:52:15,689 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000003-wal.1733237531470.temp to hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000008 2024-12-03T14:52:15,689 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T14:52:15,689 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470, journal: Splitting hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470, size=0 (0bytes) at 1733237531670Finishing writing output for hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 so closing down at 1733237535676 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000003-wal.1733237531470.temp at 1733237535679 (+3 ms)3 split writer threads finished at 1733237535679Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000003-wal.1733237531470.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733237535687 (+8 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000003-wal.1733237531470.temp to hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000008 at 1733237535689 (+2 ms)Processed 5 edits across 1 Regions in 14 ms; skipped=3; WAL=hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470, size=0, length=0, corrupted=false, cancelled=false at 1733237535689 2024-12-03T14:52:15,692 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237531470 2024-12-03T14:52:15,693 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000008 2024-12-03T14:52:15,693 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:15,695 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:15,715 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237535695, exclude list is [], retry=0 2024-12-03T14:52:15,718 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:15,718 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:15,719 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:15,720 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237535695 2024-12-03T14:52:15,721 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:44869:44869)] 2024-12-03T14:52:15,721 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d0de749bcd5f79c66a37c8be66963f1a, NAME => 'testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:15,721 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:15,721 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,721 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,723 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,724 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName a 2024-12-03T14:52:15,724 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,730 DEBUG [StoreFileOpener-d0de749bcd5f79c66a37c8be66963f1a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 82f88874c0ca44df972a7826dac791d8_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T14:52:15,730 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_ 2024-12-03T14:52:15,734 DEBUG [StoreFileOpener-d0de749bcd5f79c66a37c8be66963f1a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for b5f7118471ef4726939b28dd2f0100a9_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T14:52:15,734 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_ 2024-12-03T14:52:15,737 DEBUG [StoreFileOpener-d0de749bcd5f79c66a37c8be66963f1a-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T14:52:15,737 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_ 2024-12-03T14:52:15,741 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8 2024-12-03T14:52:15,744 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/f9afe04b97214e4e88911f37b4761dad 2024-12-03T14:52:15,745 WARN [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16adfd94 2024-12-03T14:52:15,745 WARN [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16adfd94 2024-12-03T14:52:15,745 WARN [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16adfd94 2024-12-03T14:52:15,745 WARN [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@16adfd94 2024-12-03T14:52:15,745 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8] to archive 2024-12-03T14:52:15,746 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T14:52:15,747 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_ to hdfs://localhost:38155/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/82f88874c0ca44df972a7826dac791d8_SeqId_4_ 2024-12-03T14:52:15,749 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_ to hdfs://localhost:38155/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/b5f7118471ef4726939b28dd2f0100a9_SeqId_4_ 2024-12-03T14:52:15,750 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_ to hdfs://localhost:38155/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_ 2024-12-03T14:52:15,750 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8 to hdfs://localhost:38155/hbase/archive/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/a/d55895ea68da42ba90183d0f541fc0a8 2024-12-03T14:52:15,751 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,751 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,751 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName b 2024-12-03T14:52:15,751 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,752 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,752 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,753 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0de749bcd5f79c66a37c8be66963f1a columnFamilyName c 2024-12-03T14:52:15,753 DEBUG [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,753 INFO [StoreOpener-d0de749bcd5f79c66a37c8be66963f1a-1 {}] regionserver.HStore(327): Store=d0de749bcd5f79c66a37c8be66963f1a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,753 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,754 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,756 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000008 2024-12-03T14:52:15,758 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:15,759 DEBUG [Time-limited test {}] regionserver.HRegion(5836): d0de749bcd5f79c66a37c8be66963f1a : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "d0de749bcd5f79c66a37c8be66963f1a" family_name: "a" compaction_input: "d55895ea68da42ba90183d0f541fc0a8" compaction_input: "b5f7118471ef4726939b28dd2f0100a9_SeqId_4_" compaction_input: "bd32e11b8cc0440a8bbfa87e00efc9ad_SeqId_4_" compaction_input: "82f88874c0ca44df972a7826dac791d8_SeqId_4_" compaction_output: "f9afe04b97214e4e88911f37b4761dad" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-03T14:52:15,759 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-03T14:52:15,759 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000008 2024-12-03T14:52:15,760 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/0000000000000000008 2024-12-03T14:52:15,761 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,761 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,762 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:15,763 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d0de749bcd5f79c66a37c8be66963f1a 2024-12-03T14:52:15,765 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testCompactedBulkLoadedFiles/d0de749bcd5f79c66a37c8be66963f1a/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T14:52:15,766 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d0de749bcd5f79c66a37c8be66963f1a; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74429668, jitterRate=0.1090884804725647}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:15,766 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d0de749bcd5f79c66a37c8be66963f1a: Writing region info on filesystem at 1733237535722Initializing all the Stores at 1733237535723 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535723Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535723Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535723Cleaning up temporary data from old regions at 1733237535761 (+38 ms)Region opened successfully at 1733237535766 (+5 ms) 2024-12-03T14:52:15,768 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d0de749bcd5f79c66a37c8be66963f1a, disabling compactions & flushes 2024-12-03T14:52:15,768 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:15,768 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:15,768 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. after waiting 0 ms 2024-12-03T14:52:15,768 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:15,768 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733237531415.d0de749bcd5f79c66a37c8be66963f1a. 2024-12-03T14:52:15,769 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d0de749bcd5f79c66a37c8be66963f1a: Waiting for close lock at 1733237535768Disabling compacts and flushes for region at 1733237535768Disabling writes for close at 1733237535768Writing region close event to WAL at 1733237535768Closed at 1733237535768 2024-12-03T14:52:15,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741924_1103 (size=93) 2024-12-03T14:52:15,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741924_1103 (size=93) 2024-12-03T14:52:15,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741924_1103 (size=93) 2024-12-03T14:52:15,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:52:15,773 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733237535695) 2024-12-03T14:52:15,785 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=438 (was 424) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1674911302_22 at /127.0.0.1:52708 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:38155 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1674911302_22 at /127.0.0.1:42592 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:38155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1674911302_22 at /127.0.0.1:35782 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=159 (was 138) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4573 (was 4587) 2024-12-03T14:52:15,786 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-03T14:52:15,798 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=159, ProcessCount=11, AvailableMemoryMB=4573 2024-12-03T14:52:15,798 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-03T14:52:15,811 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:15,812 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T14:52:15,813 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T14:52:15,815 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54203767, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/hregion-54203767, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:15,826 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54203767/hregion-54203767.1733237535815, exclude list is [], retry=0 2024-12-03T14:52:15,828 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:15,828 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:15,828 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:15,833 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54203767/hregion-54203767.1733237535815 2024-12-03T14:52:15,834 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:52:15,834 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 89bb48736562377ce5066b0dbff0192b, NAME => 'testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38155/hbase 2024-12-03T14:52:15,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741926_1105 (size=67) 2024-12-03T14:52:15,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741926_1105 (size=67) 2024-12-03T14:52:15,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741926_1105 (size=67) 2024-12-03T14:52:15,842 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:15,843 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,844 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName a 2024-12-03T14:52:15,844 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,845 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,845 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,846 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName b 2024-12-03T14:52:15,846 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,847 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,847 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,848 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName c 2024-12-03T14:52:15,848 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,849 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,849 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,849 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,850 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,851 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,851 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,851 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:15,852 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,854 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T14:52:15,855 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89bb48736562377ce5066b0dbff0192b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72774414, jitterRate=0.08442327380180359}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:15,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89bb48736562377ce5066b0dbff0192b: Writing region info on filesystem at 1733237535842Initializing all the Stores at 1733237535842Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535843 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535843Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535843Cleaning up temporary data from old regions at 1733237535851 (+8 ms)Region opened successfully at 1733237535855 (+4 ms) 2024-12-03T14:52:15,855 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89bb48736562377ce5066b0dbff0192b, disabling compactions & flushes 2024-12-03T14:52:15,855 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,855 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,855 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. after waiting 0 ms 2024-12-03T14:52:15,855 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,856 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,856 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89bb48736562377ce5066b0dbff0192b: Waiting for close lock at 1733237535855Disabling compacts and flushes for region at 1733237535855Disabling writes for close at 1733237535855Writing region close event to WAL at 1733237535856 (+1 ms)Closed at 1733237535856 2024-12-03T14:52:15,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741925_1104 (size=93) 2024-12-03T14:52:15,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741925_1104 (size=93) 2024-12-03T14:52:15,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741925_1104 (size=93) 2024-12-03T14:52:15,860 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:52:15,860 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54203767:(num 1733237535815) 2024-12-03T14:52:15,860 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:15,862 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:15,873 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862, exclude list is [], retry=0 2024-12-03T14:52:15,876 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:15,876 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:15,877 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:15,878 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862 2024-12-03T14:52:15,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:52:15,879 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89bb48736562377ce5066b0dbff0192b, NAME => 'testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:15,879 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:15,879 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,879 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,880 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,881 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName a 2024-12-03T14:52:15,881 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,882 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,882 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,883 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName b 2024-12-03T14:52:15,883 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,883 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,884 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,884 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName c 2024-12-03T14:52:15,884 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:15,885 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:15,885 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,886 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,887 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,888 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,888 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,888 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:15,890 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:15,891 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89bb48736562377ce5066b0dbff0192b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64550601, jitterRate=-0.03812108933925629}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:15,891 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89bb48736562377ce5066b0dbff0192b: Writing region info on filesystem at 1733237535879Initializing all the Stores at 1733237535880 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535880Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535880Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237535880Cleaning up temporary data from old regions at 1733237535888 (+8 ms)Region opened successfully at 1733237535891 (+3 ms) 2024-12-03T14:52:15,898 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89bb48736562377ce5066b0dbff0192b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T14:52:15,914 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/a/18b9b7f597d04c79935efd514094d29c is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733237535891/Put/seqid=0 2024-12-03T14:52:15,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741928_1107 (size=5958) 2024-12-03T14:52:15,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741928_1107 (size=5958) 2024-12-03T14:52:15,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741928_1107 (size=5958) 2024-12-03T14:52:15,921 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/a/18b9b7f597d04c79935efd514094d29c 2024-12-03T14:52:15,927 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/a/18b9b7f597d04c79935efd514094d29c as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/a/18b9b7f597d04c79935efd514094d29c 2024-12-03T14:52:15,931 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/a/18b9b7f597d04c79935efd514094d29c, entries=10, sequenceid=13, filesize=5.8 K 2024-12-03T14:52:15,932 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 89bb48736562377ce5066b0dbff0192b in 34ms, sequenceid=13, compaction requested=false 2024-12-03T14:52:15,932 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 89bb48736562377ce5066b0dbff0192b: 2024-12-03T14:52:15,950 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89bb48736562377ce5066b0dbff0192b, disabling compactions & flushes 2024-12-03T14:52:15,950 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,950 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,950 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. after waiting 0 ms 2024-12-03T14:52:15,950 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,952 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,952 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:15,952 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89bb48736562377ce5066b0dbff0192b: Waiting for close lock at 1733237535950Disabling compacts and flushes for region at 1733237535950Disabling writes for close at 1733237535950Writing region close event to WAL at 1733237535952 (+2 ms)Closed at 1733237535952 2024-12-03T14:52:15,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741927_1106 (size=2805) 2024-12-03T14:52:15,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741927_1106 (size=2805) 2024-12-03T14:52:15,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741927_1106 (size=2805) 2024-12-03T14:52:15,969 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862, size=2.7 K (2805bytes) 2024-12-03T14:52:15,970 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862 2024-12-03T14:52:15,970 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862 after 0ms 2024-12-03T14:52:15,972 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:15,972 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862 took 3ms 2024-12-03T14:52:15,974 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862 so closing down 2024-12-03T14:52:15,974 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:52:15,975 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733237535862.temp 2024-12-03T14:52:15,976 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000003-wal.1733237535862.temp 2024-12-03T14:52:15,977 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:52:15,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741929_1108 (size=2312) 2024-12-03T14:52:15,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741929_1108 (size=2312) 2024-12-03T14:52:15,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741929_1108 (size=2312) 2024-12-03T14:52:15,986 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000003-wal.1733237535862.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T14:52:15,987 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000003-wal.1733237535862.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000035 2024-12-03T14:52:15,987 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-12-03T14:52:15,987 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862, journal: Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862, size=2.7 K (2805bytes) at 1733237535969Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862 so closing down at 1733237535974 (+5 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000003-wal.1733237535862.temp at 1733237535976 (+2 ms)3 split writer threads finished at 1733237535977 (+1 ms)Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000003-wal.1733237535862.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733237535986 (+9 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000003-wal.1733237535862.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000035 at 1733237535987 (+1 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1733237535987 2024-12-03T14:52:15,989 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535862 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237535862 2024-12-03T14:52:15,990 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000035 2024-12-03T14:52:15,990 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:15,991 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:16,004 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992, exclude list is [], retry=0 2024-12-03T14:52:16,006 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:16,006 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:16,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:16,008 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 2024-12-03T14:52:16,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369),(127.0.0.1/127.0.0.1:36271:36271)] 2024-12-03T14:52:16,009 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 89bb48736562377ce5066b0dbff0192b, NAME => 'testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T14:52:16,009 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:16,009 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,009 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,011 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,012 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName a 2024-12-03T14:52:16,012 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:16,017 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/a/18b9b7f597d04c79935efd514094d29c 2024-12-03T14:52:16,017 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:16,017 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,018 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName b 2024-12-03T14:52:16,018 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:16,019 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:16,019 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,020 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName c 2024-12-03T14:52:16,020 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:16,020 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:16,020 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,023 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000035 2024-12-03T14:52:16,024 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:16,025 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000035 2024-12-03T14:52:16,025 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89bb48736562377ce5066b0dbff0192b 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-03T14:52:16,039 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/b/47eccbdcb9224bbd81b077200642a10c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733237535932/Put/seqid=0 2024-12-03T14:52:16,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741931_1110 (size=5958) 2024-12-03T14:52:16,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741931_1110 (size=5958) 2024-12-03T14:52:16,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741931_1110 (size=5958) 2024-12-03T14:52:16,046 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/b/47eccbdcb9224bbd81b077200642a10c 2024-12-03T14:52:16,066 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/c/e4a9c45c6de54bb7bf7aa157895d2f69 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733237535941/Put/seqid=0 2024-12-03T14:52:16,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741932_1111 (size=5958) 2024-12-03T14:52:16,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741932_1111 (size=5958) 2024-12-03T14:52:16,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741932_1111 (size=5958) 2024-12-03T14:52:16,073 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/c/e4a9c45c6de54bb7bf7aa157895d2f69 2024-12-03T14:52:16,079 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/b/47eccbdcb9224bbd81b077200642a10c as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/b/47eccbdcb9224bbd81b077200642a10c 2024-12-03T14:52:16,084 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/b/47eccbdcb9224bbd81b077200642a10c, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T14:52:16,085 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/c/e4a9c45c6de54bb7bf7aa157895d2f69 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/c/e4a9c45c6de54bb7bf7aa157895d2f69 2024-12-03T14:52:16,090 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/c/e4a9c45c6de54bb7bf7aa157895d2f69, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T14:52:16,090 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 89bb48736562377ce5066b0dbff0192b in 65ms, sequenceid=35, compaction requested=false; wal=null 2024-12-03T14:52:16,091 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000035 2024-12-03T14:52:16,092 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,092 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,092 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:16,094 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:16,096 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-03T14:52:16,097 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89bb48736562377ce5066b0dbff0192b; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65402838, jitterRate=-0.02542176842689514}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:16,097 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89bb48736562377ce5066b0dbff0192b: Writing region info on filesystem at 1733237536009Initializing all the Stores at 1733237536010 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237536010Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237536011 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237536011Obtaining lock to block concurrent updates at 1733237536025 (+14 ms)Preparing flush snapshotting stores in 89bb48736562377ce5066b0dbff0192b at 1733237536025Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733237536025Flushing stores of testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. at 1733237536025Flushing 89bb48736562377ce5066b0dbff0192b/b: creating writer at 1733237536026 (+1 ms)Flushing 89bb48736562377ce5066b0dbff0192b/b: appending metadata at 1733237536039 (+13 ms)Flushing 89bb48736562377ce5066b0dbff0192b/b: closing flushed file at 1733237536039Flushing 89bb48736562377ce5066b0dbff0192b/c: creating writer at 1733237536050 (+11 ms)Flushing 89bb48736562377ce5066b0dbff0192b/c: appending metadata at 1733237536065 (+15 ms)Flushing 89bb48736562377ce5066b0dbff0192b/c: closing flushed file at 1733237536066 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f264b39: reopening flushed file at 1733237536078 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34542371: reopening flushed file at 1733237536084 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 89bb48736562377ce5066b0dbff0192b in 65ms, sequenceid=35, compaction requested=false; wal=null at 1733237536090 (+6 ms)Cleaning up temporary data from old regions at 1733237536092 (+2 ms)Region opened successfully at 1733237536097 (+5 ms) 2024-12-03T14:52:16,120 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T14:52:16,162 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992, size=0 (0bytes) 2024-12-03T14:52:16,162 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 might be still open, length is 0 2024-12-03T14:52:16,162 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 2024-12-03T14:52:16,163 WARN [IPC Server handler 1 on default port 38155 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-03T14:52:16,163 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 after 1ms 2024-12-03T14:52:16,290 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:35846 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40191:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35846 dst: /127.0.0.1:40191 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:40191 remote=/127.0.0.1:35846]. Total timeout mills is 60000, 59837 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:16,292 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:52758 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37353:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52758 dst: /127.0.0.1:37353 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:16,292 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1645377558_22 at /127.0.0.1:42642 [Receiving block BP-870883219-172.17.0.2-1733237498670:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:38931:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42642 dst: /127.0.0.1:38931 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:16,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741930_1112 (size=2304) 2024-12-03T14:52:16,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741930_1112 (size=2304) 2024-12-03T14:52:20,164 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 after 4002ms 2024-12-03T14:52:20,171 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:20,171 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 took 4009ms 2024-12-03T14:52:20,174 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992; continuing. 2024-12-03T14:52:20,174 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 so closing down 2024-12-03T14:52:20,174 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T14:52:20,175 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733237535992.temp 2024-12-03T14:52:20,177 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000037-wal.1733237535992.temp 2024-12-03T14:52:20,177 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T14:52:20,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741933_1113 (size=2312) 2024-12-03T14:52:20,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741933_1113 (size=2312) 2024-12-03T14:52:20,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741933_1113 (size=2312) 2024-12-03T14:52:20,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000037-wal.1733237535992.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T14:52:20,186 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000037-wal.1733237535992.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000066 2024-12-03T14:52:20,186 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T14:52:20,187 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992, journal: Splitting hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992, size=0 (0bytes) at 1733237536162Finishing writing output for hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 so closing down at 1733237540174 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000037-wal.1733237535992.temp at 1733237540177 (+3 ms)3 split writer threads finished at 1733237540177Closed recovered edits writer path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000037-wal.1733237535992.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733237540185 (+8 ms)Rename recovered edits hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000037-wal.1733237535992.temp to hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000066 at 1733237540186 (+1 ms)Processed 30 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992, size=0, length=0, corrupted=false, cancelled=false at 1733237540186 2024-12-03T14:52:20,188 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 to hdfs://localhost:38155/hbase/oldWALs/wal.1733237535992 2024-12-03T14:52:20,189 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000066 2024-12-03T14:52:20,189 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T14:52:20,191 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:38155/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810, archiveDir=hdfs://localhost:38155/hbase/oldWALs, maxLogs=32 2024-12-03T14:52:20,201 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237540191, exclude list is [], retry=0 2024-12-03T14:52:20,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38931,DS-03cba0f8-f5dc-4bcd-a5a0-9918e6a66505,DISK] 2024-12-03T14:52:20,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40191,DS-110a8131-74a0-42b4-9d36-f29cd08198b9,DISK] 2024-12-03T14:52:20,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37353,DS-64dc720f-7496-41d5-bed7-d1f64d66c987,DISK] 2024-12-03T14:52:20,207 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237540191 2024-12-03T14:52:20,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36271:36271),(127.0.0.1/127.0.0.1:44869:44869),(127.0.0.1/127.0.0.1:36369:36369)] 2024-12-03T14:52:20,208 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T14:52:20,209 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,210 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName a 2024-12-03T14:52:20,210 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:20,215 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/a/18b9b7f597d04c79935efd514094d29c 2024-12-03T14:52:20,215 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:20,215 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,216 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName b 2024-12-03T14:52:20,216 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:20,220 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/b/47eccbdcb9224bbd81b077200642a10c 2024-12-03T14:52:20,220 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:20,221 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,221 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 89bb48736562377ce5066b0dbff0192b columnFamilyName c 2024-12-03T14:52:20,221 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T14:52:20,227 DEBUG [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/c/e4a9c45c6de54bb7bf7aa157895d2f69 2024-12-03T14:52:20,227 INFO [StoreOpener-89bb48736562377ce5066b0dbff0192b-1 {}] regionserver.HStore(327): Store=89bb48736562377ce5066b0dbff0192b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T14:52:20,227 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,228 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,229 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,230 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000066 2024-12-03T14:52:20,232 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-12-03T14:52:20,237 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000066 2024-12-03T14:52:20,237 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 89bb48736562377ce5066b0dbff0192b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T14:52:20,251 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/a/3312a8572ee24952b520575abbf735a6 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733237536106/Put/seqid=0 2024-12-03T14:52:20,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741935_1115 (size=5958) 2024-12-03T14:52:20,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741935_1115 (size=5958) 2024-12-03T14:52:20,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741935_1115 (size=5958) 2024-12-03T14:52:20,257 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/a/3312a8572ee24952b520575abbf735a6 2024-12-03T14:52:20,273 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/b/82c8912dc85e4e4db341ab47a68f9416 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733237536113/Put/seqid=0 2024-12-03T14:52:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741936_1116 (size=5958) 2024-12-03T14:52:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741936_1116 (size=5958) 2024-12-03T14:52:20,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741936_1116 (size=5958) 2024-12-03T14:52:20,279 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/b/82c8912dc85e4e4db341ab47a68f9416 2024-12-03T14:52:20,300 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/c/4de877888551479dac2d3a3e2223d090 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733237536121/Put/seqid=0 2024-12-03T14:52:20,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741937_1117 (size=5958) 2024-12-03T14:52:20,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741937_1117 (size=5958) 2024-12-03T14:52:20,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741937_1117 (size=5958) 2024-12-03T14:52:20,306 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/c/4de877888551479dac2d3a3e2223d090 2024-12-03T14:52:20,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/a/3312a8572ee24952b520575abbf735a6 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/a/3312a8572ee24952b520575abbf735a6 2024-12-03T14:52:20,316 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/a/3312a8572ee24952b520575abbf735a6, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T14:52:20,316 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/b/82c8912dc85e4e4db341ab47a68f9416 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/b/82c8912dc85e4e4db341ab47a68f9416 2024-12-03T14:52:20,320 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/b/82c8912dc85e4e4db341ab47a68f9416, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T14:52:20,321 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/.tmp/c/4de877888551479dac2d3a3e2223d090 as hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/c/4de877888551479dac2d3a3e2223d090 2024-12-03T14:52:20,325 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/c/4de877888551479dac2d3a3e2223d090, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T14:52:20,326 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 89bb48736562377ce5066b0dbff0192b in 88ms, sequenceid=66, compaction requested=false; wal=null 2024-12-03T14:52:20,326 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/0000000000000000066 2024-12-03T14:52:20,327 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,327 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,328 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T14:52:20,329 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 89bb48736562377ce5066b0dbff0192b 2024-12-03T14:52:20,331 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/hbase/data/default/testReplayEditsWrittenViaHRegion/89bb48736562377ce5066b0dbff0192b/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-03T14:52:20,332 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 89bb48736562377ce5066b0dbff0192b; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63742203, jitterRate=-0.050167158246040344}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T14:52:20,333 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 89bb48736562377ce5066b0dbff0192b: Writing region info on filesystem at 1733237540208Initializing all the Stores at 1733237540209 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237540209Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237540209Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733237540209Obtaining lock to block concurrent updates at 1733237540237 (+28 ms)Preparing flush snapshotting stores in 89bb48736562377ce5066b0dbff0192b at 1733237540237Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733237540237Flushing stores of testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. at 1733237540237Flushing 89bb48736562377ce5066b0dbff0192b/a: creating writer at 1733237540237Flushing 89bb48736562377ce5066b0dbff0192b/a: appending metadata at 1733237540250 (+13 ms)Flushing 89bb48736562377ce5066b0dbff0192b/a: closing flushed file at 1733237540250Flushing 89bb48736562377ce5066b0dbff0192b/b: creating writer at 1733237540260 (+10 ms)Flushing 89bb48736562377ce5066b0dbff0192b/b: appending metadata at 1733237540272 (+12 ms)Flushing 89bb48736562377ce5066b0dbff0192b/b: closing flushed file at 1733237540272Flushing 89bb48736562377ce5066b0dbff0192b/c: creating writer at 1733237540283 (+11 ms)Flushing 89bb48736562377ce5066b0dbff0192b/c: appending metadata at 1733237540299 (+16 ms)Flushing 89bb48736562377ce5066b0dbff0192b/c: closing flushed file at 1733237540299Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c64dd26: reopening flushed file at 1733237540310 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ea7f8b5: reopening flushed file at 1733237540316 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a7de0c3: reopening flushed file at 1733237540321 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 89bb48736562377ce5066b0dbff0192b in 88ms, sequenceid=66, compaction requested=false; wal=null at 1733237540326 (+5 ms)Cleaning up temporary data from old regions at 1733237540327 (+1 ms)Region opened successfully at 1733237540333 (+6 ms) 2024-12-03T14:52:20,346 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 89bb48736562377ce5066b0dbff0192b, disabling compactions & flushes 2024-12-03T14:52:20,346 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:20,346 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:20,346 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. after waiting 0 ms 2024-12-03T14:52:20,346 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:20,347 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733237535811.89bb48736562377ce5066b0dbff0192b. 2024-12-03T14:52:20,347 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 89bb48736562377ce5066b0dbff0192b: Waiting for close lock at 1733237540346Disabling compacts and flushes for region at 1733237540346Disabling writes for close at 1733237540346Writing region close event to WAL at 1733237540347 (+1 ms)Closed at 1733237540347 2024-12-03T14:52:20,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741934_1114 (size=93) 2024-12-03T14:52:20,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741934_1114 (size=93) 2024-12-03T14:52:20,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741934_1114 (size=93) 2024-12-03T14:52:20,351 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T14:52:20,351 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733237540191) 2024-12-03T14:52:20,362 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=446 (was 438) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:38155 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_838424958_22 at /127.0.0.1:58280 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_838424958_22 at /127.0.0.1:59422 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_838424958_22 at /127.0.0.1:38772 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:38155 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=146 (was 159), ProcessCount=11 (was 11), AvailableMemoryMB=4554 (was 4573) 2024-12-03T14:52:20,363 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-03T14:52:20,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T14:52:20,363 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:52:20,363 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:52:20,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:52:20,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:52:20,363 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T14:52:20,364 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T14:52:20,364 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=315704162, stopped=false 2024-12-03T14:52:20,364 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a5d22df9eca2,46647,1733237501335 2024-12-03T14:52:20,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T14:52:20,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T14:52:20,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:52:20,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T14:52:20,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:52:20,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:52:20,365 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T14:52:20,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:52:20,366 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T14:52:20,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:52:20,366 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:52:20,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T14:52:20,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:52:20,366 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,43671,1733237501988' ***** 2024-12-03T14:52:20,366 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T14:52:20,366 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a5d22df9eca2,38259,1733237502024' ***** 2024-12-03T14:52:20,366 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T14:52:20,367 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(3091): Received CLOSE for cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,43671,1733237501988 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;a5d22df9eca2:43671. 2024-12-03T14:52:20,367 DEBUG [RS:1;a5d22df9eca2:43671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:52:20,367 DEBUG [RS:1;a5d22df9eca2:43671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(959): stopping server a5d22df9eca2,38259,1733237502024 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;a5d22df9eca2:38259. 2024-12-03T14:52:20,367 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T14:52:20,367 DEBUG [RS:2;a5d22df9eca2:38259 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T14:52:20,367 DEBUG [RS:2;a5d22df9eca2:38259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:52:20,367 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T14:52:20,367 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cfbb889b168bb28ed91fe90346363f81, disabling compactions & flushes 2024-12-03T14:52:20,367 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T14:52:20,367 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1325): Online Regions={cfbb889b168bb28ed91fe90346363f81=testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81.} 2024-12-03T14:52:20,367 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:20,368 DEBUG [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1351): Waiting on cfbb889b168bb28ed91fe90346363f81 2024-12-03T14:52:20,368 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:20,368 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. after waiting 0 ms 2024-12-03T14:52:20,368 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:20,368 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T14:52:20,368 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T14:52:20,368 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T14:52:20,368 DEBUG [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T14:52:20,368 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T14:52:20,368 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T14:52:20,368 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T14:52:20,368 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T14:52:20,368 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T14:52:20,368 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-03T14:52:20,372 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/default/testReplayEditsAfterRegionMovedWithMultiCF/cfbb889b168bb28ed91fe90346363f81/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-03T14:52:20,373 INFO [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:20,373 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cfbb889b168bb28ed91fe90346363f81: Waiting for close lock at 1733237540367Running coprocessor pre-close hooks at 1733237540367Disabling compacts and flushes for region at 1733237540367Disabling writes for close at 1733237540368 (+1 ms)Writing region close event to WAL at 1733237540369 (+1 ms)Running coprocessor post-close hooks at 1733237540373 (+4 ms)Closed at 1733237540373 2024-12-03T14:52:20,373 DEBUG [RS_CLOSE_REGION-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81. 2024-12-03T14:52:20,386 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/info/a83a82bf70ab42c98188c3674a3ec3dc is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733237517526.cfbb889b168bb28ed91fe90346363f81./info:regioninfo/1733237520580/Put/seqid=0 2024-12-03T14:52:20,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741938_1118 (size=8243) 2024-12-03T14:52:20,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741938_1118 (size=8243) 2024-12-03T14:52:20,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741938_1118 (size=8243) 2024-12-03T14:52:20,393 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/info/a83a82bf70ab42c98188c3674a3ec3dc 2024-12-03T14:52:20,411 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/ns/7de82e0d067c4780a2d4571d820f4b06 is 43, key is default/ns:d/1733237503850/Put/seqid=0 2024-12-03T14:52:20,411 INFO [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:52:20,411 INFO [regionserver/a5d22df9eca2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:52:20,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741939_1119 (size=5153) 2024-12-03T14:52:20,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741939_1119 (size=5153) 2024-12-03T14:52:20,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741939_1119 (size=5153) 2024-12-03T14:52:20,417 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/ns/7de82e0d067c4780a2d4571d820f4b06 2024-12-03T14:52:20,434 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/table/4c99f7b7f2654ce09be877f7b864de6b is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733237517988/Put/seqid=0 2024-12-03T14:52:20,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741940_1120 (size=5431) 2024-12-03T14:52:20,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741940_1120 (size=5431) 2024-12-03T14:52:20,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741940_1120 (size=5431) 2024-12-03T14:52:20,440 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/table/4c99f7b7f2654ce09be877f7b864de6b 2024-12-03T14:52:20,446 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/info/a83a82bf70ab42c98188c3674a3ec3dc as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/info/a83a82bf70ab42c98188c3674a3ec3dc 2024-12-03T14:52:20,452 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/info/a83a82bf70ab42c98188c3674a3ec3dc, entries=18, sequenceid=21, filesize=8.0 K 2024-12-03T14:52:20,453 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/ns/7de82e0d067c4780a2d4571d820f4b06 as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/ns/7de82e0d067c4780a2d4571d820f4b06 2024-12-03T14:52:20,458 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/ns/7de82e0d067c4780a2d4571d820f4b06, entries=2, sequenceid=21, filesize=5.0 K 2024-12-03T14:52:20,460 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/.tmp/table/4c99f7b7f2654ce09be877f7b864de6b as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/table/4c99f7b7f2654ce09be877f7b864de6b 2024-12-03T14:52:20,464 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/table/4c99f7b7f2654ce09be877f7b864de6b, entries=2, sequenceid=21, filesize=5.3 K 2024-12-03T14:52:20,465 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=21, compaction requested=false 2024-12-03T14:52:20,469 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-03T14:52:20,470 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T14:52:20,470 INFO [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T14:52:20,470 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733237540368Running coprocessor pre-close hooks at 1733237540368Disabling compacts and flushes for region at 1733237540368Disabling writes for close at 1733237540368Obtaining lock to block concurrent updates at 1733237540368Preparing flush snapshotting stores in 1588230740 at 1733237540368Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733237540369 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733237540369Flushing 1588230740/info: creating writer at 1733237540370 (+1 ms)Flushing 1588230740/info: appending metadata at 1733237540385 (+15 ms)Flushing 1588230740/info: closing flushed file at 1733237540385Flushing 1588230740/ns: creating writer at 1733237540398 (+13 ms)Flushing 1588230740/ns: appending metadata at 1733237540410 (+12 ms)Flushing 1588230740/ns: closing flushed file at 1733237540410Flushing 1588230740/table: creating writer at 1733237540421 (+11 ms)Flushing 1588230740/table: appending metadata at 1733237540433 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733237540433Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12aa3a24: reopening flushed file at 1733237540445 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d3b8452: reopening flushed file at 1733237540452 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a6d6d: reopening flushed file at 1733237540459 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 97ms, sequenceid=21, compaction requested=false at 1733237540465 (+6 ms)Writing region close event to WAL at 1733237540466 (+1 ms)Running coprocessor post-close hooks at 1733237540470 (+4 ms)Closed at 1733237540470 2024-12-03T14:52:20,470 DEBUG [RS_CLOSE_META-regionserver/a5d22df9eca2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T14:52:20,568 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,38259,1733237502024; all regions closed. 2024-12-03T14:52:20,568 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(976): stopping server a5d22df9eca2,43671,1733237501988; all regions closed. 2024-12-03T14:52:20,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741834_1010 (size=841) 2024-12-03T14:52:20,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741834_1010 (size=841) 2024-12-03T14:52:20,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741834_1010 (size=841) 2024-12-03T14:52:20,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741836_1012 (size=3783) 2024-12-03T14:52:20,572 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988/a5d22df9eca2%2C43671%2C1733237501988.meta.1733237503698.meta not finished, retry = 0 2024-12-03T14:52:20,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741836_1012 (size=3783) 2024-12-03T14:52:20,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741836_1012 (size=3783) 2024-12-03T14:52:20,574 DEBUG [RS:2;a5d22df9eca2:38259 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs 2024-12-03T14:52:20,574 INFO [RS:2;a5d22df9eca2:38259 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a5d22df9eca2%2C38259%2C1733237502024:(num 1733237503225) 2024-12-03T14:52:20,574 DEBUG [RS:2;a5d22df9eca2:38259 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:52:20,574 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:52:20,574 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:52:20,574 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T14:52:20,575 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T14:52:20,575 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T14:52:20,575 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T14:52:20,575 INFO [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:52:20,575 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:52:20,575 INFO [RS:2;a5d22df9eca2:38259 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38259 2024-12-03T14:52:20,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,38259,1733237502024 2024-12-03T14:52:20,576 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:52:20,576 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:52:20,578 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,38259,1733237502024] 2024-12-03T14:52:20,578 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,38259,1733237502024 already deleted, retry=false 2024-12-03T14:52:20,579 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,38259,1733237502024 expired; onlineServers=1 2024-12-03T14:52:20,677 DEBUG [RS:1;a5d22df9eca2:43671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs 2024-12-03T14:52:20,678 INFO [RS:1;a5d22df9eca2:43671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a5d22df9eca2%2C43671%2C1733237501988.meta:.meta(num 1733237503698) 2024-12-03T14:52:20,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:20,678 INFO [RS:2;a5d22df9eca2:38259 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:52:20,678 INFO [RS:2;a5d22df9eca2:38259 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,38259,1733237502024; zookeeper connection closed. 2024-12-03T14:52:20,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38259-0x100a090c71d0003, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:20,679 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3fd995a0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3fd995a0 2024-12-03T14:52:20,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741833_1009 (size=2054) 2024-12-03T14:52:20,684 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/WALs/a5d22df9eca2,43671,1733237501988/a5d22df9eca2%2C43671%2C1733237501988.1733237503225 not finished, retry = 0 2024-12-03T14:52:20,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741833_1009 (size=2054) 2024-12-03T14:52:20,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741833_1009 (size=2054) 2024-12-03T14:52:20,789 DEBUG [RS:1;a5d22df9eca2:43671 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/oldWALs 2024-12-03T14:52:20,789 INFO [RS:1;a5d22df9eca2:43671 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a5d22df9eca2%2C43671%2C1733237501988:(num 1733237503225) 2024-12-03T14:52:20,789 DEBUG [RS:1;a5d22df9eca2:43671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T14:52:20,789 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T14:52:20,789 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:52:20,790 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.ChoreService(370): Chore service for: regionserver/a5d22df9eca2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T14:52:20,790 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:52:20,790 INFO [regionserver/a5d22df9eca2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:52:20,791 INFO [RS:1;a5d22df9eca2:43671 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43671 2024-12-03T14:52:20,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T14:52:20,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a5d22df9eca2,43671,1733237501988 2024-12-03T14:52:20,793 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:52:20,793 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a5d22df9eca2,43671,1733237501988] 2024-12-03T14:52:20,794 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a5d22df9eca2,43671,1733237501988 already deleted, retry=false 2024-12-03T14:52:20,794 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a5d22df9eca2,43671,1733237501988 expired; onlineServers=0 2024-12-03T14:52:20,794 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a5d22df9eca2,46647,1733237501335' ***** 2024-12-03T14:52:20,794 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T14:52:20,795 INFO [M:0;a5d22df9eca2:46647 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T14:52:20,795 INFO [M:0;a5d22df9eca2:46647 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T14:52:20,795 DEBUG [M:0;a5d22df9eca2:46647 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T14:52:20,795 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T14:52:20,795 DEBUG [M:0;a5d22df9eca2:46647 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T14:52:20,795 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237502949 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.large.0-1733237502949,5,FailOnTimeoutGroup] 2024-12-03T14:52:20,795 DEBUG [master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237502950 {}] cleaner.HFileCleaner(306): Exit Thread[master/a5d22df9eca2:0:becomeActiveMaster-HFileCleaner.small.0-1733237502950,5,FailOnTimeoutGroup] 2024-12-03T14:52:20,795 INFO [M:0;a5d22df9eca2:46647 {}] hbase.ChoreService(370): Chore service for: master/a5d22df9eca2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T14:52:20,795 INFO [M:0;a5d22df9eca2:46647 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T14:52:20,795 DEBUG [M:0;a5d22df9eca2:46647 {}] master.HMaster(1795): Stopping service threads 2024-12-03T14:52:20,795 INFO [M:0;a5d22df9eca2:46647 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T14:52:20,795 INFO [M:0;a5d22df9eca2:46647 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T14:52:20,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T14:52:20,796 INFO [M:0;a5d22df9eca2:46647 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T14:52:20,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T14:52:20,796 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T14:52:20,797 DEBUG [M:0;a5d22df9eca2:46647 {}] zookeeper.ZKUtil(347): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T14:52:20,797 WARN [M:0;a5d22df9eca2:46647 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T14:52:20,797 INFO [M:0;a5d22df9eca2:46647 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/.lastflushedseqids 2024-12-03T14:52:20,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741941_1121 (size=138) 2024-12-03T14:52:20,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741941_1121 (size=138) 2024-12-03T14:52:20,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741941_1121 (size=138) 2024-12-03T14:52:20,809 INFO [M:0;a5d22df9eca2:46647 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T14:52:20,809 INFO [M:0;a5d22df9eca2:46647 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T14:52:20,810 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T14:52:20,810 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:52:20,810 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:52:20,810 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T14:52:20,810 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:52:20,810 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-12-03T14:52:20,823 DEBUG [M:0;a5d22df9eca2:46647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e86cdbd1c4d2491b9d37aec7c3432811 is 82, key is hbase:meta,,1/info:regioninfo/1733237503775/Put/seqid=0 2024-12-03T14:52:20,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741942_1122 (size=5672) 2024-12-03T14:52:20,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741942_1122 (size=5672) 2024-12-03T14:52:20,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741942_1122 (size=5672) 2024-12-03T14:52:20,831 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e86cdbd1c4d2491b9d37aec7c3432811 2024-12-03T14:52:20,849 DEBUG [M:0;a5d22df9eca2:46647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edee46ac2dbc4a74bf950fc9fe97754f is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733237517994/Put/seqid=0 2024-12-03T14:52:20,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741943_1123 (size=7756) 2024-12-03T14:52:20,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741943_1123 (size=7756) 2024-12-03T14:52:20,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741943_1123 (size=7756) 2024-12-03T14:52:20,855 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edee46ac2dbc4a74bf950fc9fe97754f 2024-12-03T14:52:20,859 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for edee46ac2dbc4a74bf950fc9fe97754f 2024-12-03T14:52:20,872 DEBUG [M:0;a5d22df9eca2:46647 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9aaac892b5554d32831a37080d0f23a9 is 69, key is a5d22df9eca2,38259,1733237502024/rs:state/1733237502993/Put/seqid=0 2024-12-03T14:52:20,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741944_1124 (size=5445) 2024-12-03T14:52:20,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741944_1124 (size=5445) 2024-12-03T14:52:20,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741944_1124 (size=5445) 2024-12-03T14:52:20,878 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9aaac892b5554d32831a37080d0f23a9 2024-12-03T14:52:20,883 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9aaac892b5554d32831a37080d0f23a9 2024-12-03T14:52:20,884 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e86cdbd1c4d2491b9d37aec7c3432811 as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e86cdbd1c4d2491b9d37aec7c3432811 2024-12-03T14:52:20,889 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e86cdbd1c4d2491b9d37aec7c3432811, entries=8, sequenceid=168, filesize=5.5 K 2024-12-03T14:52:20,890 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edee46ac2dbc4a74bf950fc9fe97754f as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/edee46ac2dbc4a74bf950fc9fe97754f 2024-12-03T14:52:20,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:20,894 INFO [RS:1;a5d22df9eca2:43671 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:52:20,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43671-0x100a090c71d0002, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:20,894 INFO [RS:1;a5d22df9eca2:43671 {}] regionserver.HRegionServer(1031): Exiting; stopping=a5d22df9eca2,43671,1733237501988; zookeeper connection closed. 2024-12-03T14:52:20,894 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c795a1a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c795a1a 2024-12-03T14:52:20,894 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T14:52:20,895 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for edee46ac2dbc4a74bf950fc9fe97754f 2024-12-03T14:52:20,895 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/edee46ac2dbc4a74bf950fc9fe97754f, entries=17, sequenceid=168, filesize=7.6 K 2024-12-03T14:52:20,896 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9aaac892b5554d32831a37080d0f23a9 as hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9aaac892b5554d32831a37080d0f23a9 2024-12-03T14:52:20,901 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9aaac892b5554d32831a37080d0f23a9 2024-12-03T14:52:20,901 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38155/user/jenkins/test-data/a09668e7-7c61-bc9b-7f49-cef7ba0a1918/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9aaac892b5554d32831a37080d0f23a9, entries=3, sequenceid=168, filesize=5.3 K 2024-12-03T14:52:20,902 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false 2024-12-03T14:52:20,904 INFO [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T14:52:20,904 DEBUG [M:0;a5d22df9eca2:46647 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733237540810Disabling compacts and flushes for region at 1733237540810Disabling writes for close at 1733237540810Obtaining lock to block concurrent updates at 1733237540810Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733237540810Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1733237540810Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733237540811 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733237540811Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733237540823 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733237540823Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733237540835 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733237540848 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733237540849 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733237540859 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733237540872 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733237540872Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d960775: reopening flushed file at 1733237540883 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5daafa2e: reopening flushed file at 1733237540889 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@571a3476: reopening flushed file at 1733237540895 (+6 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false at 1733237540903 (+8 ms)Writing region close event to WAL at 1733237540904 (+1 ms)Closed at 1733237540904 2024-12-03T14:52:20,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40191 is added to blk_1073741830_1006 (size=71349) 2024-12-03T14:52:20,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38931 is added to blk_1073741830_1006 (size=71349) 2024-12-03T14:52:20,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37353 is added to blk_1073741830_1006 (size=71349) 2024-12-03T14:52:20,907 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T14:52:20,907 INFO [M:0;a5d22df9eca2:46647 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T14:52:20,907 INFO [M:0;a5d22df9eca2:46647 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46647 2024-12-03T14:52:20,907 INFO [M:0;a5d22df9eca2:46647 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T14:52:21,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:21,009 INFO [M:0;a5d22df9eca2:46647 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T14:52:21,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46647-0x100a090c71d0000, quorum=127.0.0.1:51554, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T14:52:21,018 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521231 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521231 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733237520949/wal.1733237521231 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T14:52:21,019 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733237512594/wal.1733237512706 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:21,019 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733237535810/wal.1733237535992 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:21,021 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237530653 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237530653 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733237521386/wal.1733237530653 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T14:52:21,022 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733237504819/wal.1733237504886 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:21,024 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237531271 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237531271 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733237530844/wal.1733237531271 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T14:52:21,024 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733237531414/wal.1733237531470 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T14:52:21,026 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal.1733237504696 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal.1733237504696 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733237504532/wal.1733237504696 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T14:52:21,028 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal.1733237504276 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal.1733237504276 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733237504047/wal.1733237504276 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1645377558_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-12-03T14:52:21,032 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26f8cb79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:52:21,034 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:52:21,034 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:52:21,034 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:52:21,034 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,STOPPED} 2024-12-03T14:52:21,036 WARN [BP-870883219-172.17.0.2-1733237498670 heartbeating to localhost/127.0.0.1:38155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T14:52:21,036 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T14:52:21,036 WARN [BP-870883219-172.17.0.2-1733237498670 heartbeating to localhost/127.0.0.1:38155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-870883219-172.17.0.2-1733237498670 (Datanode Uuid b785c90a-24ce-4fac-ac2d-3b2d6c761cec) service to localhost/127.0.0.1:38155 2024-12-03T14:52:21,036 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T14:52:21,037 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data5/current/BP-870883219-172.17.0.2-1733237498670 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:52:21,037 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data6/current/BP-870883219-172.17.0.2-1733237498670 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:52:21,038 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T14:52:21,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4464725f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:52:21,055 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:52:21,055 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:52:21,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:52:21,055 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,STOPPED} 2024-12-03T14:52:21,056 WARN [BP-870883219-172.17.0.2-1733237498670 heartbeating to localhost/127.0.0.1:38155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T14:52:21,056 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T14:52:21,056 WARN [BP-870883219-172.17.0.2-1733237498670 heartbeating to localhost/127.0.0.1:38155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-870883219-172.17.0.2-1733237498670 (Datanode Uuid 85a55e3f-948b-4b5c-a683-9fce9149759b) service to localhost/127.0.0.1:38155 2024-12-03T14:52:21,056 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T14:52:21,057 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data3/current/BP-870883219-172.17.0.2-1733237498670 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:52:21,057 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data4/current/BP-870883219-172.17.0.2-1733237498670 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:52:21,057 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T14:52:21,059 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17d1a8d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T14:52:21,059 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:52:21,059 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:52:21,059 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:52:21,059 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,STOPPED} 2024-12-03T14:52:21,060 WARN [BP-870883219-172.17.0.2-1733237498670 heartbeating to localhost/127.0.0.1:38155 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T14:52:21,060 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T14:52:21,060 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T14:52:21,060 WARN [BP-870883219-172.17.0.2-1733237498670 heartbeating to localhost/127.0.0.1:38155 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-870883219-172.17.0.2-1733237498670 (Datanode Uuid c42b874a-68d0-4d7c-b8c3-72f99028c692) service to localhost/127.0.0.1:38155 2024-12-03T14:52:21,061 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data1/current/BP-870883219-172.17.0.2-1733237498670 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:52:21,061 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/cluster_c4294d91-85ed-5637-bc64-818e68e85af0/data/data2/current/BP-870883219-172.17.0.2-1733237498670 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T14:52:21,061 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T14:52:21,066 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c8b857b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T14:52:21,067 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T14:52:21,067 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T14:52:21,067 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T14:52:21,067 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/38977cbe-7dfe-af76-a9c6-744121b9f53e/hadoop.log.dir/,STOPPED} 2024-12-03T14:52:21,075 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T14:52:21,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down