2024-11-14 20:29:57,786 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-14 20:29:57,802 main DEBUG Took 0.013420 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-14 20:29:57,803 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-14 20:29:57,803 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-14 20:29:57,804 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-14 20:29:57,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,815 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-14 20:29:57,830 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,833 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,836 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,836 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,837 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,838 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,838 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,839 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,840 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,840 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,841 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,841 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,842 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,842 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,843 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,843 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,844 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,844 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,845 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,845 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,846 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,846 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-14 20:29:57,847 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,847 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-14 20:29:57,849 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-14 20:29:57,850 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-14 20:29:57,852 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-14 20:29:57,853 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-14 20:29:57,854 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-14 20:29:57,855 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-14 20:29:57,867 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-14 20:29:57,870 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-14 20:29:57,872 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-14 20:29:57,872 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-14 20:29:57,873 main DEBUG createAppenders(={Console}) 2024-11-14 20:29:57,874 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-14 20:29:57,874 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-14 20:29:57,875 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-14 20:29:57,875 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-14 20:29:57,876 main DEBUG OutputStream closed 2024-11-14 20:29:57,876 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-14 20:29:57,876 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-14 20:29:57,877 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-14 20:29:57,966 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-14 20:29:57,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-14 20:29:57,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-14 20:29:57,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-14 20:29:57,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-14 20:29:57,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-14 20:29:57,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-14 20:29:57,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-14 20:29:57,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-14 20:29:57,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-14 20:29:57,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-14 20:29:57,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-14 20:29:57,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-14 20:29:57,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-14 20:29:57,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-14 20:29:57,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-14 20:29:57,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-14 20:29:57,976 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-14 20:29:57,979 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14 20:29:57,979 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-14 20:29:57,979 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-14 20:29:57,980 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-14T20:29:58,331 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38 2024-11-14 20:29:58,334 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-14 20:29:58,335 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-14T20:29:58,348 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-14T20:29:58,356 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-11-14T20:29:58,387 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-14T20:29:58,452 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-14T20:29:58,452 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-14T20:29:58,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-14T20:29:58,489 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4, deleteOnExit=true 2024-11-14T20:29:58,489 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-14T20:29:58,491 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/test.cache.data in system properties and HBase conf 2024-11-14T20:29:58,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.tmp.dir in system properties and HBase conf 2024-11-14T20:29:58,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir in system properties and HBase conf 2024-11-14T20:29:58,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-14T20:29:58,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-14T20:29:58,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-14T20:29:58,607 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-14T20:29:58,750 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-14T20:29:58,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-14T20:29:58,754 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-14T20:29:58,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-14T20:29:58,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T20:29:58,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-14T20:29:58,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-14T20:29:58,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-14T20:29:58,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T20:29:58,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-14T20:29:58,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/nfs.dump.dir in system properties and HBase conf 2024-11-14T20:29:58,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/java.io.tmpdir in system properties and HBase conf 2024-11-14T20:29:58,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-14T20:29:58,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-14T20:29:58,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-14T20:29:59,762 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-14T20:29:59,863 INFO [Time-limited test {}] log.Log(170): Logging initialized @3137ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-14T20:29:59,964 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T20:30:00,039 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T20:30:00,072 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T20:30:00,072 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T20:30:00,074 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T20:30:00,102 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T20:30:00,105 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,AVAILABLE} 2024-11-14T20:30:00,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T20:30:00,315 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3717288f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/java.io.tmpdir/jetty-localhost-38239-hadoop-hdfs-3_4_1-tests_jar-_-any-383428539144566162/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T20:30:00,324 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:38239} 2024-11-14T20:30:00,324 INFO [Time-limited test {}] server.Server(415): Started @3598ms 2024-11-14T20:30:00,781 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T20:30:00,792 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T20:30:00,795 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T20:30:00,795 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T20:30:00,795 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-14T20:30:00,796 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,AVAILABLE} 2024-11-14T20:30:00,797 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T20:30:00,930 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@330740de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/java.io.tmpdir/jetty-localhost-36625-hadoop-hdfs-3_4_1-tests_jar-_-any-4686054719390611378/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T20:30:00,931 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:36625} 2024-11-14T20:30:00,931 INFO [Time-limited test {}] server.Server(415): Started @4205ms 2024-11-14T20:30:00,989 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T20:30:01,134 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T20:30:01,141 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T20:30:01,146 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T20:30:01,146 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T20:30:01,146 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T20:30:01,148 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,AVAILABLE} 2024-11-14T20:30:01,149 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T20:30:01,333 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7bd427b8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/java.io.tmpdir/jetty-localhost-40121-hadoop-hdfs-3_4_1-tests_jar-_-any-16931416541364646900/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T20:30:01,334 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:40121} 2024-11-14T20:30:01,334 INFO [Time-limited test {}] server.Server(415): Started @4608ms 2024-11-14T20:30:01,337 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T20:30:01,395 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-14T20:30:01,402 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-14T20:30:01,406 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-14T20:30:01,406 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-14T20:30:01,406 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-14T20:30:01,407 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,AVAILABLE} 2024-11-14T20:30:01,408 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-14T20:30:01,633 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@35f1150e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/java.io.tmpdir/jetty-localhost-41545-hadoop-hdfs-3_4_1-tests_jar-_-any-367883673624380152/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T20:30:01,633 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data1/current/BP-866344823-172.17.0.3-1731616199461/current, will proceed with Du for space computation calculation, 2024-11-14T20:30:01,636 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:41545} 2024-11-14T20:30:01,636 INFO [Time-limited test {}] server.Server(415): Started @4910ms 2024-11-14T20:30:01,637 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data3/current/BP-866344823-172.17.0.3-1731616199461/current, will proceed with Du for space computation calculation, 2024-11-14T20:30:01,644 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-14T20:30:01,645 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data2/current/BP-866344823-172.17.0.3-1731616199461/current, will proceed with Du for space computation calculation, 2024-11-14T20:30:01,646 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data4/current/BP-866344823-172.17.0.3-1731616199461/current, will proceed with Du for space computation calculation, 2024-11-14T20:30:01,726 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T20:30:01,744 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T20:30:01,836 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1a99092edb8c0b7 with lease ID 0xf944ffe14d1fc66b: Processing first storage report for DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782 from datanode DatanodeRegistration(127.0.0.1:41701, datanodeUuid=dd75bad4-f7b2-4f9a-9203-8adb2036a1e3, infoPort=39289, infoSecurePort=0, ipcPort=43777, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461) 2024-11-14T20:30:01,839 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1a99092edb8c0b7 with lease ID 0xf944ffe14d1fc66b: from storage DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782 node DatanodeRegistration(127.0.0.1:41701, datanodeUuid=dd75bad4-f7b2-4f9a-9203-8adb2036a1e3, infoPort=39289, infoSecurePort=0, ipcPort=43777, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461), blocks: 0, hasStaleStorage: true, processing time: 4 msecs, invalidatedBlocks: 0 2024-11-14T20:30:01,840 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc1a176cdf5fdc4f2 with lease ID 0xf944ffe14d1fc66a: Processing first storage report for DS-829e2125-1cf6-4abe-92e7-10d5c245b269 from datanode DatanodeRegistration(127.0.0.1:41951, datanodeUuid=1428fba7-9712-4cef-8ee1-65176d395f14, infoPort=43695, infoSecurePort=0, ipcPort=38913, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461) 2024-11-14T20:30:01,840 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc1a176cdf5fdc4f2 with lease ID 0xf944ffe14d1fc66a: from storage DS-829e2125-1cf6-4abe-92e7-10d5c245b269 node DatanodeRegistration(127.0.0.1:41951, datanodeUuid=1428fba7-9712-4cef-8ee1-65176d395f14, infoPort=43695, infoSecurePort=0, ipcPort=38913, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T20:30:01,841 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd1a99092edb8c0b7 with lease ID 0xf944ffe14d1fc66b: Processing first storage report for DS-6369f44b-09ba-4a47-9f27-144603c80b8a from datanode DatanodeRegistration(127.0.0.1:41701, datanodeUuid=dd75bad4-f7b2-4f9a-9203-8adb2036a1e3, infoPort=39289, infoSecurePort=0, ipcPort=43777, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461) 2024-11-14T20:30:01,841 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd1a99092edb8c0b7 with lease ID 0xf944ffe14d1fc66b: from storage DS-6369f44b-09ba-4a47-9f27-144603c80b8a node DatanodeRegistration(127.0.0.1:41701, datanodeUuid=dd75bad4-f7b2-4f9a-9203-8adb2036a1e3, infoPort=39289, infoSecurePort=0, ipcPort=43777, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T20:30:01,841 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc1a176cdf5fdc4f2 with lease ID 0xf944ffe14d1fc66a: Processing first storage report for DS-4994fbfa-4a99-4b03-b354-d047d73726ba from datanode DatanodeRegistration(127.0.0.1:41951, datanodeUuid=1428fba7-9712-4cef-8ee1-65176d395f14, infoPort=43695, infoSecurePort=0, ipcPort=38913, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461) 2024-11-14T20:30:01,841 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc1a176cdf5fdc4f2 with lease ID 0xf944ffe14d1fc66a: from storage DS-4994fbfa-4a99-4b03-b354-d047d73726ba node DatanodeRegistration(127.0.0.1:41951, datanodeUuid=1428fba7-9712-4cef-8ee1-65176d395f14, infoPort=43695, infoSecurePort=0, ipcPort=38913, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T20:30:01,894 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data6/current/BP-866344823-172.17.0.3-1731616199461/current, will proceed with Du for space computation calculation, 2024-11-14T20:30:01,894 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data5/current/BP-866344823-172.17.0.3-1731616199461/current, will proceed with Du for space computation calculation, 2024-11-14T20:30:01,940 WARN [Thread-126 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-14T20:30:01,947 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb729fffdd186cec4 with lease ID 0xf944ffe14d1fc66c: Processing first storage report for DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c from datanode DatanodeRegistration(127.0.0.1:37797, datanodeUuid=bfdfaea9-1ab9-4521-ba18-c32481d43c60, infoPort=42955, infoSecurePort=0, ipcPort=40899, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461) 2024-11-14T20:30:01,947 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb729fffdd186cec4 with lease ID 0xf944ffe14d1fc66c: from storage DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c node DatanodeRegistration(127.0.0.1:37797, datanodeUuid=bfdfaea9-1ab9-4521-ba18-c32481d43c60, infoPort=42955, infoSecurePort=0, ipcPort=40899, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-14T20:30:01,948 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb729fffdd186cec4 with lease ID 0xf944ffe14d1fc66c: Processing first storage report for DS-69f0f528-a281-4645-8824-61815d78cd15 from datanode DatanodeRegistration(127.0.0.1:37797, datanodeUuid=bfdfaea9-1ab9-4521-ba18-c32481d43c60, infoPort=42955, infoSecurePort=0, ipcPort=40899, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461) 2024-11-14T20:30:01,948 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb729fffdd186cec4 with lease ID 0xf944ffe14d1fc66c: from storage DS-69f0f528-a281-4645-8824-61815d78cd15 node DatanodeRegistration(127.0.0.1:37797, datanodeUuid=bfdfaea9-1ab9-4521-ba18-c32481d43c60, infoPort=42955, infoSecurePort=0, ipcPort=40899, storageInfo=lv=-57;cid=testClusterID;nsid=1696257646;c=1731616199461), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-14T20:30:02,252 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38 2024-11-14T20:30:02,354 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/zookeeper_0, clientPort=60776, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-14T20:30:02,368 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60776 2024-11-14T20:30:02,384 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:02,388 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:02,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741825_1001 (size=7) 2024-11-14T20:30:02,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741825_1001 (size=7) 2024-11-14T20:30:02,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741825_1001 (size=7) 2024-11-14T20:30:03,099 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668 with version=8 2024-11-14T20:30:03,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/hbase-staging 2024-11-14T20:30:03,524 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6c8768110af2:0 server-side Connection retries=45 2024-11-14T20:30:03,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:03,536 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:03,543 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T20:30:03,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:03,543 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T20:30:03,760 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-14T20:30:03,838 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-14T20:30:03,854 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-14T20:30:03,860 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T20:30:03,915 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 11299 (auto-detected) 2024-11-14T20:30:03,916 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-14T20:30:03,959 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36945 2024-11-14T20:30:03,985 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36945 connecting to ZooKeeper ensemble=127.0.0.1:60776 2024-11-14T20:30:04,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:369450x0, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T20:30:04,038 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36945-0x1003ffc70d80000 connected 2024-11-14T20:30:04,068 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,072 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,086 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T20:30:04,091 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668, hbase.cluster.distributed=false 2024-11-14T20:30:04,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T20:30:04,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36945 2024-11-14T20:30:04,127 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36945 2024-11-14T20:30:04,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36945 2024-11-14T20:30:04,128 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36945 2024-11-14T20:30:04,130 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36945 2024-11-14T20:30:04,259 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6c8768110af2:0 server-side Connection retries=45 2024-11-14T20:30:04,260 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,261 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T20:30:04,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,261 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T20:30:04,264 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T20:30:04,266 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T20:30:04,267 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46509 2024-11-14T20:30:04,270 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46509 connecting to ZooKeeper ensemble=127.0.0.1:60776 2024-11-14T20:30:04,272 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,275 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:465090x0, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T20:30:04,285 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:465090x0, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T20:30:04,290 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T20:30:04,293 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46509-0x1003ffc70d80001 connected 2024-11-14T20:30:04,305 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T20:30:04,308 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T20:30:04,317 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T20:30:04,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46509 2024-11-14T20:30:04,321 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46509 2024-11-14T20:30:04,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46509 2024-11-14T20:30:04,322 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46509 2024-11-14T20:30:04,323 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46509 2024-11-14T20:30:04,352 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6c8768110af2:0 server-side Connection retries=45 2024-11-14T20:30:04,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,352 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,353 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T20:30:04,353 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,354 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T20:30:04,354 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T20:30:04,354 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T20:30:04,357 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44657 2024-11-14T20:30:04,359 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44657 connecting to ZooKeeper ensemble=127.0.0.1:60776 2024-11-14T20:30:04,360 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,362 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,376 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446570x0, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T20:30:04,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:446570x0, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T20:30:04,377 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T20:30:04,381 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44657-0x1003ffc70d80002 connected 2024-11-14T20:30:04,384 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T20:30:04,385 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T20:30:04,388 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T20:30:04,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44657 2024-11-14T20:30:04,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44657 2024-11-14T20:30:04,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44657 2024-11-14T20:30:04,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44657 2024-11-14T20:30:04,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44657 2024-11-14T20:30:04,417 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6c8768110af2:0 server-side Connection retries=45 2024-11-14T20:30:04,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,418 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-14T20:30:04,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-14T20:30:04,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-14T20:30:04,419 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-14T20:30:04,419 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-14T20:30:04,425 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42163 2024-11-14T20:30:04,427 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42163 connecting to ZooKeeper ensemble=127.0.0.1:60776 2024-11-14T20:30:04,429 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,432 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421630x0, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-14T20:30:04,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42163-0x1003ffc70d80003 connected 2024-11-14T20:30:04,442 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T20:30:04,442 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-14T20:30:04,443 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-14T20:30:04,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-14T20:30:04,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-14T20:30:04,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42163 2024-11-14T20:30:04,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42163 2024-11-14T20:30:04,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42163 2024-11-14T20:30:04,467 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42163 2024-11-14T20:30:04,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42163 2024-11-14T20:30:04,492 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6c8768110af2:36945 2024-11-14T20:30:04,493 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6c8768110af2,36945,1731616203247 2024-11-14T20:30:04,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,513 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6c8768110af2,36945,1731616203247 2024-11-14T20:30:04,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T20:30:04,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T20:30:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-14T20:30:04,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,560 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-14T20:30:04,564 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6c8768110af2,36945,1731616203247 from backup master directory 2024-11-14T20:30:04,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6c8768110af2,36945,1731616203247 2024-11-14T20:30:04,569 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,571 WARN [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T20:30:04,571 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6c8768110af2,36945,1731616203247 2024-11-14T20:30:04,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-14T20:30:04,573 INFO [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-14T20:30:04,575 INFO [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-14T20:30:04,680 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/hbase.id] with ID: 0a234c43-7201-4428-9c4c-8a6f53c029dd 2024-11-14T20:30:04,680 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/.tmp/hbase.id 2024-11-14T20:30:04,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741826_1002 (size=42) 2024-11-14T20:30:04,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741826_1002 (size=42) 2024-11-14T20:30:04,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741826_1002 (size=42) 2024-11-14T20:30:04,710 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/.tmp/hbase.id]:[hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/hbase.id] 2024-11-14T20:30:04,786 INFO [master/6c8768110af2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-14T20:30:04,792 INFO [master/6c8768110af2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-14T20:30:04,827 INFO [master/6c8768110af2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 29ms. 2024-11-14T20:30:04,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:04,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741827_1003 (size=196) 2024-11-14T20:30:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741827_1003 (size=196) 2024-11-14T20:30:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741827_1003 (size=196) 2024-11-14T20:30:04,905 INFO [master/6c8768110af2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T20:30:04,908 INFO [master/6c8768110af2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-14T20:30:04,916 INFO [master/6c8768110af2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:04,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741828_1004 (size=1189) 2024-11-14T20:30:04,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741828_1004 (size=1189) 2024-11-14T20:30:04,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741828_1004 (size=1189) 2024-11-14T20:30:05,004 INFO [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store 2024-11-14T20:30:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741829_1005 (size=34) 2024-11-14T20:30:05,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741829_1005 (size=34) 2024-11-14T20:30:05,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741829_1005 (size=34) 2024-11-14T20:30:05,461 INFO [master/6c8768110af2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-14T20:30:05,466 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:05,467 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T20:30:05,467 INFO [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:05,468 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:05,469 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T20:30:05,470 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:05,470 INFO [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:05,471 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731616205467Disabling compacts and flushes for region at 1731616205467Disabling writes for close at 1731616205469 (+2 ms)Writing region close event to WAL at 1731616205470 (+1 ms)Closed at 1731616205470 2024-11-14T20:30:05,474 WARN [master/6c8768110af2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/.initializing 2024-11-14T20:30:05,474 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/WALs/6c8768110af2,36945,1731616203247 2024-11-14T20:30:05,485 INFO [master/6c8768110af2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:05,504 INFO [master/6c8768110af2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6c8768110af2%2C36945%2C1731616203247, suffix=, logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/WALs/6c8768110af2,36945,1731616203247, archiveDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/oldWALs, maxLogs=10 2024-11-14T20:30:05,548 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/WALs/6c8768110af2,36945,1731616203247/6c8768110af2%2C36945%2C1731616203247.1731616205510, exclude list is [], retry=0 2024-11-14T20:30:05,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:05,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:05,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:05,583 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:05,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-14T20:30:05,639 INFO [master/6c8768110af2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/WALs/6c8768110af2,36945,1731616203247/6c8768110af2%2C36945%2C1731616203247.1731616205510 2024-11-14T20:30:05,640 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:05,640 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:05,641 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:05,645 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,646 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,732 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-14T20:30:05,737 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:05,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:05,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-14T20:30:05,745 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:05,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:05,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-14T20:30:05,750 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:05,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:05,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-14T20:30:05,755 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:05,756 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:05,756 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,760 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,762 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,771 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,772 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,777 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T20:30:05,781 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-14T20:30:05,787 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:05,789 INFO [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64823378, jitterRate=-0.034056395292282104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T20:30:05,798 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731616205664Initializing all the Stores at 1731616205666 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616205667 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616205668 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616205668Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616205668Cleaning up temporary data from old regions at 1731616205772 (+104 ms)Region opened successfully at 1731616205798 (+26 ms) 2024-11-14T20:30:05,800 INFO [master/6c8768110af2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-14T20:30:05,867 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ddea217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6c8768110af2/172.17.0.3:0 2024-11-14T20:30:05,913 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-14T20:30:05,928 INFO [master/6c8768110af2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-14T20:30:05,929 INFO [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-14T20:30:05,932 INFO [master/6c8768110af2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-14T20:30:05,934 INFO [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-14T20:30:05,941 INFO [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-14T20:30:05,941 INFO [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-14T20:30:05,978 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-14T20:30:05,990 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-14T20:30:05,992 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-14T20:30:05,995 INFO [master/6c8768110af2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-14T20:30:05,997 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-14T20:30:05,999 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-14T20:30:06,002 INFO [master/6c8768110af2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-14T20:30:06,006 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-14T20:30:06,008 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-14T20:30:06,010 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-14T20:30:06,012 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-14T20:30:06,041 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-14T20:30:06,042 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-14T20:30:06,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T20:30:06,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T20:30:06,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T20:30:06,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-14T20:30:06,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,052 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6c8768110af2,36945,1731616203247, sessionid=0x1003ffc70d80000, setting cluster-up flag (Was=false) 2024-11-14T20:30:06,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,107 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-14T20:30:06,113 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6c8768110af2,36945,1731616203247 2024-11-14T20:30:06,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:06,128 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-14T20:30:06,130 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6c8768110af2,36945,1731616203247 2024-11-14T20:30:06,138 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-14T20:30:06,174 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(746): ClusterId : 0a234c43-7201-4428-9c4c-8a6f53c029dd 2024-11-14T20:30:06,174 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(746): ClusterId : 0a234c43-7201-4428-9c4c-8a6f53c029dd 2024-11-14T20:30:06,175 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(746): ClusterId : 0a234c43-7201-4428-9c4c-8a6f53c029dd 2024-11-14T20:30:06,178 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T20:30:06,178 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T20:30:06,178 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-14T20:30:06,187 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T20:30:06,187 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T20:30:06,187 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-14T20:30:06,187 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T20:30:06,187 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T20:30:06,187 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-14T20:30:06,191 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T20:30:06,191 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T20:30:06,192 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-14T20:30:06,192 DEBUG [RS:0;6c8768110af2:46509 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@367f7959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6c8768110af2/172.17.0.3:0 2024-11-14T20:30:06,192 DEBUG [RS:2;6c8768110af2:42163 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69704fc5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6c8768110af2/172.17.0.3:0 2024-11-14T20:30:06,193 DEBUG [RS:1;6c8768110af2:44657 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a77e60b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6c8768110af2/172.17.0.3:0 2024-11-14T20:30:06,220 DEBUG [RS:1;6c8768110af2:44657 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6c8768110af2:44657 2024-11-14T20:30:06,226 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6c8768110af2:46509 2024-11-14T20:30:06,238 INFO [RS:0;6c8768110af2:46509 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T20:30:06,238 INFO [RS:0;6c8768110af2:46509 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T20:30:06,238 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T20:30:06,238 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;6c8768110af2:42163 2024-11-14T20:30:06,239 INFO [RS:2;6c8768110af2:42163 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T20:30:06,239 INFO [RS:2;6c8768110af2:42163 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T20:30:06,239 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T20:30:06,242 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(2659): reportForDuty to master=6c8768110af2,36945,1731616203247 with port=46509, startcode=1731616204216 2024-11-14T20:30:06,244 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(2659): reportForDuty to master=6c8768110af2,36945,1731616203247 with port=42163, startcode=1731616204417 2024-11-14T20:30:06,249 INFO [RS:1;6c8768110af2:44657 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-14T20:30:06,249 INFO [RS:1;6c8768110af2:44657 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-14T20:30:06,249 DEBUG [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-14T20:30:06,254 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(2659): reportForDuty to master=6c8768110af2,36945,1731616203247 with port=44657, startcode=1731616204351 2024-11-14T20:30:06,259 INFO [AsyncFSWAL-0-hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData-prefix:6c8768110af2,36945,1731616203247 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-11-14T20:30:06,269 DEBUG [RS:0;6c8768110af2:46509 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T20:30:06,269 DEBUG [RS:2;6c8768110af2:42163 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T20:30:06,269 DEBUG [RS:1;6c8768110af2:44657 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-14T20:30:06,282 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-14T20:30:06,298 INFO [master/6c8768110af2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-14T20:30:06,311 INFO [master/6c8768110af2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-14T20:30:06,321 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6c8768110af2,36945,1731616203247 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-14T20:30:06,331 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6c8768110af2:0, corePoolSize=5, maxPoolSize=5 2024-11-14T20:30:06,332 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6c8768110af2:0, corePoolSize=5, maxPoolSize=5 2024-11-14T20:30:06,333 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6c8768110af2:0, corePoolSize=5, maxPoolSize=5 2024-11-14T20:30:06,333 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6c8768110af2:0, corePoolSize=5, maxPoolSize=5 2024-11-14T20:30:06,333 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6c8768110af2:0, corePoolSize=10, maxPoolSize=10 2024-11-14T20:30:06,334 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,334 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6c8768110af2:0, corePoolSize=2, maxPoolSize=2 2024-11-14T20:30:06,334 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,362 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33603, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T20:30:06,362 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55383, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T20:30:06,364 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32949, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-14T20:30:06,377 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T20:30:06,382 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T20:30:06,383 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-14T20:30:06,395 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T20:30:06,396 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-14T20:30:06,397 INFO [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731616236397 2024-11-14T20:30:06,400 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-14T20:30:06,401 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-14T20:30:06,412 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:06,412 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-14T20:30:06,412 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-14T20:30:06,413 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-14T20:30:06,413 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-14T20:30:06,413 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-14T20:30:06,415 DEBUG [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T20:30:06,415 WARN [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T20:30:06,415 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T20:30:06,416 WARN [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T20:30:06,416 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-14T20:30:06,416 WARN [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-14T20:30:06,416 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,423 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-14T20:30:06,424 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-14T20:30:06,425 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-14T20:30:06,438 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-14T20:30:06,439 INFO [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-14T20:30:06,444 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6c8768110af2:0:becomeActiveMaster-HFileCleaner.large.0-1731616206441,5,FailOnTimeoutGroup] 2024-11-14T20:30:06,456 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6c8768110af2:0:becomeActiveMaster-HFileCleaner.small.0-1731616206445,5,FailOnTimeoutGroup] 2024-11-14T20:30:06,456 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,456 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-14T20:30:06,458 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,458 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741831_1007 (size=1321) 2024-11-14T20:30:06,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741831_1007 (size=1321) 2024-11-14T20:30:06,519 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(2659): reportForDuty to master=6c8768110af2,36945,1731616203247 with port=46509, startcode=1731616204216 2024-11-14T20:30:06,521 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(2659): reportForDuty to master=6c8768110af2,36945,1731616203247 with port=42163, startcode=1731616204417 2024-11-14T20:30:06,522 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6c8768110af2,46509,1731616204216 2024-11-14T20:30:06,523 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(2659): reportForDuty to master=6c8768110af2,36945,1731616203247 with port=44657, startcode=1731616204351 2024-11-14T20:30:06,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741831_1007 (size=1321) 2024-11-14T20:30:06,529 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] master.ServerManager(517): Registering regionserver=6c8768110af2,46509,1731616204216 2024-11-14T20:30:06,531 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-14T20:30:06,532 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668 2024-11-14T20:30:06,545 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6c8768110af2,42163,1731616204417 2024-11-14T20:30:06,545 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668 2024-11-14T20:30:06,545 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43677 2024-11-14T20:30:06,545 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T20:30:06,546 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] master.ServerManager(517): Registering regionserver=6c8768110af2,42163,1731616204417 2024-11-14T20:30:06,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T20:30:06,551 DEBUG [RS:0;6c8768110af2:46509 {}] zookeeper.ZKUtil(111): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6c8768110af2,46509,1731616204216 2024-11-14T20:30:06,552 WARN [RS:0;6c8768110af2:46509 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T20:30:06,552 INFO [RS:0;6c8768110af2:46509 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:06,553 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668 2024-11-14T20:30:06,553 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43677 2024-11-14T20:30:06,553 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T20:30:06,553 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6c8768110af2,44657,1731616204351 2024-11-14T20:30:06,554 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36945 {}] master.ServerManager(517): Registering regionserver=6c8768110af2,44657,1731616204351 2024-11-14T20:30:06,552 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216 2024-11-14T20:30:06,565 DEBUG [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668 2024-11-14T20:30:06,565 DEBUG [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43677 2024-11-14T20:30:06,565 DEBUG [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-14T20:30:06,566 DEBUG [RS:2;6c8768110af2:42163 {}] zookeeper.ZKUtil(111): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6c8768110af2,42163,1731616204417 2024-11-14T20:30:06,567 WARN [RS:2;6c8768110af2:42163 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T20:30:06,567 INFO [RS:2;6c8768110af2:42163 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:06,567 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417 2024-11-14T20:30:06,568 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T20:30:06,569 DEBUG [RS:1;6c8768110af2:44657 {}] zookeeper.ZKUtil(111): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6c8768110af2,44657,1731616204351 2024-11-14T20:30:06,569 WARN [RS:1;6c8768110af2:44657 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-14T20:30:06,569 INFO [RS:1;6c8768110af2:44657 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:06,569 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6c8768110af2,46509,1731616204216] 2024-11-14T20:30:06,569 DEBUG [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,44657,1731616204351 2024-11-14T20:30:06,569 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6c8768110af2,42163,1731616204417] 2024-11-14T20:30:06,571 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6c8768110af2,44657,1731616204351] 2024-11-14T20:30:06,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741832_1008 (size=32) 2024-11-14T20:30:06,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741832_1008 (size=32) 2024-11-14T20:30:06,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741832_1008 (size=32) 2024-11-14T20:30:06,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:06,593 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T20:30:06,600 INFO [RS:0;6c8768110af2:46509 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T20:30:06,600 INFO [RS:1;6c8768110af2:44657 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T20:30:06,605 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T20:30:06,605 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:06,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:06,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T20:30:06,614 INFO [RS:2;6c8768110af2:42163 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-14T20:30:06,622 INFO [RS:2;6c8768110af2:42163 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T20:30:06,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T20:30:06,623 INFO [RS:1;6c8768110af2:44657 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T20:30:06,623 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:06,623 INFO [RS:0;6c8768110af2:46509 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-14T20:30:06,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:06,624 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T20:30:06,627 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T20:30:06,627 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:06,628 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:06,629 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T20:30:06,632 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T20:30:06,632 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:06,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:06,633 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T20:30:06,635 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740 2024-11-14T20:30:06,637 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740 2024-11-14T20:30:06,641 INFO [RS:0;6c8768110af2:46509 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T20:30:06,641 INFO [RS:2;6c8768110af2:42163 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T20:30:06,641 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,641 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,641 INFO [RS:1;6c8768110af2:44657 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T20:30:06,642 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T20:30:06,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T20:30:06,644 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T20:30:06,645 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T20:30:06,647 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T20:30:06,648 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T20:30:06,652 INFO [RS:0;6c8768110af2:46509 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T20:30:06,652 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-14T20:30:06,654 INFO [RS:1;6c8768110af2:44657 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T20:30:06,654 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,654 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,654 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,654 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,654 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,654 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0, corePoolSize=2, maxPoolSize=2 2024-11-14T20:30:06,655 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0, corePoolSize=2, maxPoolSize=2 2024-11-14T20:30:06,655 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,655 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,656 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6c8768110af2:0, corePoolSize=3, maxPoolSize=3 2024-11-14T20:30:06,657 INFO [RS:2;6c8768110af2:42163 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-14T20:30:06,657 DEBUG [RS:1;6c8768110af2:44657 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6c8768110af2:0, corePoolSize=3, maxPoolSize=3 2024-11-14T20:30:06,657 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6c8768110af2:0, corePoolSize=3, maxPoolSize=3 2024-11-14T20:30:06,657 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,657 DEBUG [RS:0;6c8768110af2:46509 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6c8768110af2:0, corePoolSize=3, maxPoolSize=3 2024-11-14T20:30:06,657 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,657 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,657 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,658 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,658 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,658 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0, corePoolSize=2, maxPoolSize=2 2024-11-14T20:30:06,658 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,659 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,659 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,659 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,659 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,659 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6c8768110af2:0, corePoolSize=1, maxPoolSize=1 2024-11-14T20:30:06,659 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6c8768110af2:0, corePoolSize=3, maxPoolSize=3 2024-11-14T20:30:06,660 DEBUG [RS:2;6c8768110af2:42163 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6c8768110af2:0, corePoolSize=3, maxPoolSize=3 2024-11-14T20:30:06,661 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:06,662 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72238814, jitterRate=0.07644221186637878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T20:30:06,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731616206591Initializing all the Stores at 1731616206593 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616206593Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616206593Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616206593Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616206593Cleaning up temporary data from old regions at 1731616206643 (+50 ms)Region opened successfully at 1731616206664 (+21 ms) 2024-11-14T20:30:06,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T20:30:06,665 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T20:30:06,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T20:30:06,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T20:30:06,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T20:30:06,666 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,666 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,666 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,666 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,667 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,667 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,42163,1731616204417-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T20:30:06,670 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,670 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T20:30:06,670 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,670 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,670 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,670 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,670 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731616206664Disabling compacts and flushes for region at 1731616206664Disabling writes for close at 1731616206665 (+1 ms)Writing region close event to WAL at 1731616206669 (+4 ms)Closed at 1731616206670 (+1 ms) 2024-11-14T20:30:06,670 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,670 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,670 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,671 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,44657,1731616204351-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T20:30:06,670 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,671 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,671 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,46509,1731616204216-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T20:30:06,680 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T20:30:06,680 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-14T20:30:06,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T20:30:06,708 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T20:30:06,713 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-14T20:30:06,724 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T20:30:06,724 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T20:30:06,726 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,42163,1731616204417-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,727 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,727 INFO [RS:2;6c8768110af2:42163 {}] regionserver.Replication(171): 6c8768110af2,42163,1731616204417 started 2024-11-14T20:30:06,728 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,46509,1731616204216-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,729 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,729 INFO [RS:0;6c8768110af2:46509 {}] regionserver.Replication(171): 6c8768110af2,46509,1731616204216 started 2024-11-14T20:30:06,729 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-14T20:30:06,732 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,44657,1731616204351-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,733 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,733 INFO [RS:1;6c8768110af2:44657 {}] regionserver.Replication(171): 6c8768110af2,44657,1731616204351 started 2024-11-14T20:30:06,754 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,755 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1482): Serving as 6c8768110af2,46509,1731616204216, RpcServer on 6c8768110af2/172.17.0.3:46509, sessionid=0x1003ffc70d80001 2024-11-14T20:30:06,756 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T20:30:06,756 DEBUG [RS:0;6c8768110af2:46509 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6c8768110af2,46509,1731616204216 2024-11-14T20:30:06,757 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6c8768110af2,46509,1731616204216' 2024-11-14T20:30:06,758 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T20:30:06,759 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T20:30:06,759 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T20:30:06,760 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T20:30:06,760 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,760 DEBUG [RS:0;6c8768110af2:46509 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6c8768110af2,46509,1731616204216 2024-11-14T20:30:06,760 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6c8768110af2,46509,1731616204216' 2024-11-14T20:30:06,760 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T20:30:06,760 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1482): Serving as 6c8768110af2,42163,1731616204417, RpcServer on 6c8768110af2/172.17.0.3:42163, sessionid=0x1003ffc70d80003 2024-11-14T20:30:06,761 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T20:30:06,761 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T20:30:06,761 DEBUG [RS:2;6c8768110af2:42163 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6c8768110af2,42163,1731616204417 2024-11-14T20:30:06,761 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6c8768110af2,42163,1731616204417' 2024-11-14T20:30:06,761 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T20:30:06,761 DEBUG [RS:0;6c8768110af2:46509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T20:30:06,762 INFO [RS:0;6c8768110af2:46509 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T20:30:06,762 INFO [RS:0;6c8768110af2:46509 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T20:30:06,762 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T20:30:06,763 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T20:30:06,763 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T20:30:06,763 DEBUG [RS:2;6c8768110af2:42163 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6c8768110af2,42163,1731616204417 2024-11-14T20:30:06,763 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6c8768110af2,42163,1731616204417' 2024-11-14T20:30:06,763 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T20:30:06,764 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T20:30:06,765 DEBUG [RS:2;6c8768110af2:42163 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T20:30:06,765 INFO [RS:2;6c8768110af2:42163 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T20:30:06,766 INFO [RS:2;6c8768110af2:42163 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T20:30:06,767 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:06,768 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(1482): Serving as 6c8768110af2,44657,1731616204351, RpcServer on 6c8768110af2/172.17.0.3:44657, sessionid=0x1003ffc70d80002 2024-11-14T20:30:06,768 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-14T20:30:06,768 DEBUG [RS:1;6c8768110af2:44657 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6c8768110af2,44657,1731616204351 2024-11-14T20:30:06,768 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6c8768110af2,44657,1731616204351' 2024-11-14T20:30:06,768 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-14T20:30:06,769 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-14T20:30:06,771 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-14T20:30:06,771 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-14T20:30:06,771 DEBUG [RS:1;6c8768110af2:44657 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6c8768110af2,44657,1731616204351 2024-11-14T20:30:06,771 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6c8768110af2,44657,1731616204351' 2024-11-14T20:30:06,771 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-14T20:30:06,772 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-14T20:30:06,775 DEBUG [RS:1;6c8768110af2:44657 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-14T20:30:06,775 INFO [RS:1;6c8768110af2:44657 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-14T20:30:06,775 INFO [RS:1;6c8768110af2:44657 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-14T20:30:06,865 WARN [6c8768110af2:36945 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-14T20:30:06,869 INFO [RS:2;6c8768110af2:42163 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:06,872 INFO [RS:0;6c8768110af2:46509 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:06,874 INFO [RS:2;6c8768110af2:42163 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6c8768110af2%2C42163%2C1731616204417, suffix=, logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417, archiveDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs, maxLogs=32 2024-11-14T20:30:06,876 INFO [RS:1;6c8768110af2:44657 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:06,881 INFO [RS:0;6c8768110af2:46509 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6c8768110af2%2C46509%2C1731616204216, suffix=, logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216, archiveDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs, maxLogs=32 2024-11-14T20:30:06,884 INFO [RS:1;6c8768110af2:44657 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6c8768110af2%2C44657%2C1731616204351, suffix=, logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,44657,1731616204351, archiveDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs, maxLogs=32 2024-11-14T20:30:06,905 DEBUG [RS:2;6c8768110af2:42163 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417/6c8768110af2%2C42163%2C1731616204417.1731616206877, exclude list is [], retry=0 2024-11-14T20:30:06,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:06,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:06,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:06,914 DEBUG [RS:0;6c8768110af2:46509 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216/6c8768110af2%2C46509%2C1731616204216.1731616206883, exclude list is [], retry=0 2024-11-14T20:30:06,914 DEBUG [RS:1;6c8768110af2:44657 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,44657,1731616204351/6c8768110af2%2C44657%2C1731616204351.1731616206886, exclude list is [], retry=0 2024-11-14T20:30:06,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:06,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:06,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:06,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:06,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:06,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:06,973 INFO [RS:2;6c8768110af2:42163 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417/6c8768110af2%2C42163%2C1731616204417.1731616206877 2024-11-14T20:30:06,978 INFO [RS:1;6c8768110af2:44657 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,44657,1731616204351/6c8768110af2%2C44657%2C1731616204351.1731616206886 2024-11-14T20:30:06,978 DEBUG [RS:2;6c8768110af2:42163 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:06,985 DEBUG [RS:1;6c8768110af2:44657 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:06,988 INFO [RS:0;6c8768110af2:46509 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216/6c8768110af2%2C46509%2C1731616204216.1731616206883 2024-11-14T20:30:06,989 DEBUG [RS:0;6c8768110af2:46509 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:07,118 DEBUG [6c8768110af2:36945 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-14T20:30:07,129 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(204): Hosts are {6c8768110af2=0} racks are {/default-rack=0} 2024-11-14T20:30:07,138 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T20:30:07,138 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T20:30:07,138 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T20:30:07,138 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T20:30:07,138 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T20:30:07,138 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T20:30:07,138 INFO [6c8768110af2:36945 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T20:30:07,138 INFO [6c8768110af2:36945 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T20:30:07,138 INFO [6c8768110af2:36945 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T20:30:07,138 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T20:30:07,147 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6c8768110af2,42163,1731616204417 2024-11-14T20:30:07,153 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6c8768110af2,42163,1731616204417, state=OPENING 2024-11-14T20:30:07,159 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-14T20:30:07,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:07,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:07,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:07,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:07,165 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,165 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,165 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,167 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T20:30:07,169 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6c8768110af2,42163,1731616204417}] 2024-11-14T20:30:07,354 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T20:30:07,357 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34691, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T20:30:07,372 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T20:30:07,373 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:07,373 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T20:30:07,377 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6c8768110af2%2C42163%2C1731616204417.meta, suffix=.meta, logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417, archiveDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs, maxLogs=32 2024-11-14T20:30:07,400 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, exclude list is [], retry=0 2024-11-14T20:30:07,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:07,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:07,405 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:07,411 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta 2024-11-14T20:30:07,412 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:07,412 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:07,414 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T20:30:07,418 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T20:30:07,424 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T20:30:07,429 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T20:30:07,430 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:07,430 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T20:30:07,430 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T20:30:07,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T20:30:07,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T20:30:07,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:07,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:07,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T20:30:07,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T20:30:07,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:07,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:07,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T20:30:07,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T20:30:07,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:07,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:07,445 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T20:30:07,446 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T20:30:07,446 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:07,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:07,447 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T20:30:07,456 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740 2024-11-14T20:30:07,459 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740 2024-11-14T20:30:07,463 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T20:30:07,463 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T20:30:07,464 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T20:30:07,467 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T20:30:07,469 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68031831, jitterRate=0.01375328004360199}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T20:30:07,469 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T20:30:07,470 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731616207431Writing region info on filesystem at 1731616207431Initializing all the Stores at 1731616207435 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616207435Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616207436 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616207436Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616207436Cleaning up temporary data from old regions at 1731616207463 (+27 ms)Running coprocessor post-open hooks at 1731616207469 (+6 ms)Region opened successfully at 1731616207470 (+1 ms) 2024-11-14T20:30:07,480 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731616207342 2024-11-14T20:30:07,498 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T20:30:07,499 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T20:30:07,500 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6c8768110af2,42163,1731616204417 2024-11-14T20:30:07,506 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6c8768110af2,42163,1731616204417, state=OPEN 2024-11-14T20:30:07,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:07,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:07,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:07,509 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:07,509 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,510 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,511 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6c8768110af2,42163,1731616204417 2024-11-14T20:30:07,512 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:07,524 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-14T20:30:07,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6c8768110af2,42163,1731616204417 in 342 msec 2024-11-14T20:30:07,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-14T20:30:07,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 839 msec 2024-11-14T20:30:07,547 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-14T20:30:07,547 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-14T20:30:07,582 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:07,583 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1] 2024-11-14T20:30:07,610 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T20:30:07,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36841, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T20:30:07,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4600 sec 2024-11-14T20:30:07,645 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731616207645, completionTime=-1 2024-11-14T20:30:07,649 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-14T20:30:07,649 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-14T20:30:07,690 INFO [master/6c8768110af2:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-14T20:30:07,690 INFO [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731616267690 2024-11-14T20:30:07,690 INFO [master/6c8768110af2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731616327690 2024-11-14T20:30:07,690 INFO [master/6c8768110af2:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 40 msec 2024-11-14T20:30:07,693 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-14T20:30:07,707 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,36945,1731616203247-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:07,709 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,36945,1731616203247-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:07,709 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,36945,1731616203247-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:07,717 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6c8768110af2:36945, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:07,725 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:07,727 DEBUG [master/6c8768110af2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-14T20:30:07,729 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:07,783 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.212sec 2024-11-14T20:30:07,788 INFO [master/6c8768110af2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-14T20:30:07,790 INFO [master/6c8768110af2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-14T20:30:07,791 INFO [master/6c8768110af2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-14T20:30:07,792 INFO [master/6c8768110af2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-14T20:30:07,792 INFO [master/6c8768110af2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-14T20:30:07,793 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,36945,1731616203247-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-14T20:30:07,794 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,36945,1731616203247-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-14T20:30:07,801 DEBUG [master/6c8768110af2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-14T20:30:07,802 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-14T20:30:07,803 INFO [master/6c8768110af2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6c8768110af2,36945,1731616203247-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:07,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344137bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T20:30:07,899 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6c8768110af2,36945,-1 for getting cluster id 2024-11-14T20:30:07,903 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-14T20:30:07,916 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0a234c43-7201-4428-9c4c-8a6f53c029dd' 2024-11-14T20:30:07,918 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-14T20:30:07,918 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0a234c43-7201-4428-9c4c-8a6f53c029dd" 2024-11-14T20:30:07,918 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23f6188a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T20:30:07,919 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6c8768110af2,36945,-1] 2024-11-14T20:30:07,922 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-14T20:30:07,923 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:07,927 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-14T20:30:07,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c6d940, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-14T20:30:07,931 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:07,954 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1] 2024-11-14T20:30:07,954 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T20:30:07,963 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T20:30:08,002 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6c8768110af2,36945,1731616203247 2024-11-14T20:30:08,002 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43677/hbase 2024-11-14T20:30:08,028 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=573, ProcessCount=11, AvailableMemoryMB=3823 2024-11-14T20:30:08,060 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:08,067 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:08,072 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:08,078 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-68067904, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-68067904, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:08,103 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-68067904/hregion-68067904.1731616208080, exclude list is [], retry=0 2024-11-14T20:30:08,111 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:08,112 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:08,113 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:08,134 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-68067904/hregion-68067904.1731616208080 2024-11-14T20:30:08,137 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:08,137 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1fa2630f53990e6eede092939dc6e1ba, NAME => 'testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:08,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741838_1014 (size=64) 2024-11-14T20:30:08,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741838_1014 (size=64) 2024-11-14T20:30:08,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741838_1014 (size=64) 2024-11-14T20:30:08,165 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:08,181 INFO [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:08,184 INFO [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fa2630f53990e6eede092939dc6e1ba columnFamilyName a 2024-11-14T20:30:08,184 DEBUG [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:08,185 INFO [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] regionserver.HStore(327): Store=1fa2630f53990e6eede092939dc6e1ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:08,185 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:08,187 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:08,188 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:08,188 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:08,189 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:08,192 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:08,209 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:08,211 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1fa2630f53990e6eede092939dc6e1ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69973534, jitterRate=0.042686909437179565}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:08,214 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1fa2630f53990e6eede092939dc6e1ba: Writing region info on filesystem at 1731616208165Initializing all the Stores at 1731616208172 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616208172Cleaning up temporary data from old regions at 1731616208189 (+17 ms)Region opened successfully at 1731616208214 (+25 ms) 2024-11-14T20:30:08,214 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1fa2630f53990e6eede092939dc6e1ba, disabling compactions & flushes 2024-11-14T20:30:08,214 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba. 2024-11-14T20:30:08,215 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba. 2024-11-14T20:30:08,215 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba. after waiting 0 ms 2024-11-14T20:30:08,215 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba. 2024-11-14T20:30:08,215 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba. 2024-11-14T20:30:08,216 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1fa2630f53990e6eede092939dc6e1ba: Waiting for close lock at 1731616208214Disabling compacts and flushes for region at 1731616208214Disabling writes for close at 1731616208215 (+1 ms)Writing region close event to WAL at 1731616208215Closed at 1731616208215 2024-11-14T20:30:08,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741837_1013 (size=95) 2024-11-14T20:30:08,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741837_1013 (size=95) 2024-11-14T20:30:08,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741837_1013 (size=95) 2024-11-14T20:30:08,260 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:08,260 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-68067904:(num 1731616208080) 2024-11-14T20:30:08,262 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T20:30:08,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741839_1015 (size=320) 2024-11-14T20:30:08,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741839_1015 (size=320) 2024-11-14T20:30:08,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741839_1015 (size=320) 2024-11-14T20:30:08,319 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T20:30:08,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741840_1016 (size=253) 2024-11-14T20:30:08,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741840_1016 (size=253) 2024-11-14T20:30:08,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741840_1016 (size=253) 2024-11-14T20:30:08,843 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1, size=320 (320bytes) 2024-11-14T20:30:08,844 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-14T20:30:08,844 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-14T20:30:08,844 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1 2024-11-14T20:30:08,856 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1 after 9ms 2024-11-14T20:30:08,869 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:08,870 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1 took 30ms 2024-11-14T20:30:08,904 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1 so closing down 2024-11-14T20:30:08,904 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:08,910 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T20:30:08,914 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T20:30:08,918 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:08,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741841_1017 (size=320) 2024-11-14T20:30:08,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741841_1017 (size=320) 2024-11-14T20:30:08,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741841_1017 (size=320) 2024-11-14T20:30:08,975 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 3 ms) 2024-11-14T20:30:08,993 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002 2024-11-14T20:30:09,006 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 106 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-14T20:30:09,006 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1, journal: Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1, size=320 (320bytes) at 1731616208843Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1 so closing down at 1731616208904 (+61 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000001-wal-1.temp at 1731616208914 (+10 ms)3 split writer threads finished at 1731616208918 (+4 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 3 ms) at 1731616208975 (+57 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002 at 1731616208993 (+18 ms)Processed 2 edits across 1 Regions in 106 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1731616209006 (+13 ms) 2024-11-14T20:30:09,037 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2, size=253 (253bytes) 2024-11-14T20:30:09,037 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2 2024-11-14T20:30:09,038 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2 after 1ms 2024-11-14T20:30:09,053 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:09,054 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2 took 17ms 2024-11-14T20:30:09,070 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2 so closing down 2024-11-14T20:30:09,070 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:09,073 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T20:30:09,077 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T20:30:09,078 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:09,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741842_1018 (size=253) 2024-11-14T20:30:09,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741842_1018 (size=253) 2024-11-14T20:30:09,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741842_1018 (size=253) 2024-11-14T20:30:09,167 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:09,181 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:09,187 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-11-14T20:30:09,194 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 129 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-14T20:30:09,195 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2, journal: Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2, size=253 (253bytes) at 1731616209037Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2 so closing down at 1731616209070 (+33 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002-wal-2.temp at 1731616209077 (+7 ms)3 split writer threads finished at 1731616209078 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731616209167 (+89 ms)Processed 1 edits across 1 Regions in 129 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1731616209194 (+27 ms) 2024-11-14T20:30:09,195 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:09,205 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:09,229 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal.1731616209206, exclude list is [], retry=0 2024-11-14T20:30:09,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:09,254 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:09,255 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:09,281 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal.1731616209206 2024-11-14T20:30:09,282 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:09,283 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1fa2630f53990e6eede092939dc6e1ba, NAME => 'testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:09,283 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:09,283 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:09,283 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:09,294 INFO [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:09,309 INFO [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1fa2630f53990e6eede092939dc6e1ba columnFamilyName a 2024-11-14T20:30:09,309 DEBUG [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:09,320 INFO [StoreOpener-1fa2630f53990e6eede092939dc6e1ba-1 {}] regionserver.HStore(327): Store=1fa2630f53990e6eede092939dc6e1ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:09,321 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:09,340 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:09,357 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:09,373 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002 2024-11-14T20:30:09,386 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:09,408 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002 2024-11-14T20:30:09,438 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1fa2630f53990e6eede092939dc6e1ba 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T20:30:09,605 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/.tmp/a/331f7a3fe3504cb4b82da163dc2d0bc2 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731616208260/Put/seqid=0 2024-11-14T20:30:09,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741844_1020 (size=5170) 2024-11-14T20:30:09,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741844_1020 (size=5170) 2024-11-14T20:30:09,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741844_1020 (size=5170) 2024-11-14T20:30:10,048 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/.tmp/a/331f7a3fe3504cb4b82da163dc2d0bc2 2024-11-14T20:30:10,309 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/.tmp/a/331f7a3fe3504cb4b82da163dc2d0bc2 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/a/331f7a3fe3504cb4b82da163dc2d0bc2 2024-11-14T20:30:10,353 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/a/331f7a3fe3504cb4b82da163dc2d0bc2, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T20:30:10,362 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1fa2630f53990e6eede092939dc6e1ba in 948ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T20:30:10,368 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/0000000000000000002 2024-11-14T20:30:10,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:10,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:10,388 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1fa2630f53990e6eede092939dc6e1ba 2024-11-14T20:30:10,420 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/1fa2630f53990e6eede092939dc6e1ba/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T20:30:10,428 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1fa2630f53990e6eede092939dc6e1ba; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58896911, jitterRate=-0.12236763536930084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:10,430 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1fa2630f53990e6eede092939dc6e1ba: Writing region info on filesystem at 1731616209283Initializing all the Stores at 1731616209289 (+6 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616209289Obtaining lock to block concurrent updates at 1731616209438 (+149 ms)Preparing flush snapshotting stores in 1fa2630f53990e6eede092939dc6e1ba at 1731616209438Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731616209450 (+12 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731616208061.1fa2630f53990e6eede092939dc6e1ba. at 1731616209450Flushing 1fa2630f53990e6eede092939dc6e1ba/a: creating writer at 1731616209452 (+2 ms)Flushing 1fa2630f53990e6eede092939dc6e1ba/a: appending metadata at 1731616209575 (+123 ms)Flushing 1fa2630f53990e6eede092939dc6e1ba/a: closing flushed file at 1731616209579 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28003e10: reopening flushed file at 1731616210298 (+719 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1fa2630f53990e6eede092939dc6e1ba in 948ms, sequenceid=2, compaction requested=false; wal=null at 1731616210362 (+64 ms)Cleaning up temporary data from old regions at 1731616210370 (+8 ms)Region opened successfully at 1731616210430 (+60 ms) 2024-11-14T20:30:10,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=373 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48132 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:42182 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48124 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43677/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:57766 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:57786 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:39470 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=573 (was 573), ProcessCount=11 (was 11), AvailableMemoryMB=3678 (was 3823) 2024-11-14T20:30:10,520 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=573, ProcessCount=11, AvailableMemoryMB=3677 2024-11-14T20:30:10,550 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:10,557 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:10,560 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:10,574 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77408973, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-77408973, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:10,621 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77408973/hregion-77408973.1731616210586, exclude list is [], retry=0 2024-11-14T20:30:10,646 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:10,661 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:10,671 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:10,712 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77408973/hregion-77408973.1731616210586 2024-11-14T20:30:10,717 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:10,717 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 91e9249eb41850067c0d30835a773a0c, NAME => 'testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:10,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741846_1022 (size=64) 2024-11-14T20:30:10,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741846_1022 (size=64) 2024-11-14T20:30:10,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741846_1022 (size=64) 2024-11-14T20:30:10,822 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:10,848 INFO [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:10,853 INFO [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91e9249eb41850067c0d30835a773a0c columnFamilyName a 2024-11-14T20:30:10,853 DEBUG [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:10,855 INFO [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] regionserver.HStore(327): Store=91e9249eb41850067c0d30835a773a0c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:10,856 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:10,863 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:10,867 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:10,868 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:10,868 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:10,873 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:10,882 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:10,883 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 91e9249eb41850067c0d30835a773a0c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71972585, jitterRate=0.07247509062290192}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 91e9249eb41850067c0d30835a773a0c: Writing region info on filesystem at 1731616210822Initializing all the Stores at 1731616210840 (+18 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616210841 (+1 ms)Cleaning up temporary data from old regions at 1731616210868 (+27 ms)Region opened successfully at 1731616210883 (+15 ms) 2024-11-14T20:30:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 91e9249eb41850067c0d30835a773a0c, disabling compactions & flushes 2024-11-14T20:30:10,883 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c. 2024-11-14T20:30:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c. 2024-11-14T20:30:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c. after waiting 0 ms 2024-11-14T20:30:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c. 2024-11-14T20:30:10,885 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c. 2024-11-14T20:30:10,885 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 91e9249eb41850067c0d30835a773a0c: Waiting for close lock at 1731616210883Disabling compacts and flushes for region at 1731616210883Disabling writes for close at 1731616210883Writing region close event to WAL at 1731616210884 (+1 ms)Closed at 1731616210884 2024-11-14T20:30:10,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741845_1021 (size=95) 2024-11-14T20:30:10,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741845_1021 (size=95) 2024-11-14T20:30:10,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741845_1021 (size=95) 2024-11-14T20:30:10,915 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:10,915 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77408973:(num 1731616210586) 2024-11-14T20:30:10,916 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T20:30:10,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741847_1023 (size=320) 2024-11-14T20:30:10,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741847_1023 (size=320) 2024-11-14T20:30:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741847_1023 (size=320) 2024-11-14T20:30:10,966 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-14T20:30:11,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741848_1024 (size=253) 2024-11-14T20:30:11,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741848_1024 (size=253) 2024-11-14T20:30:11,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741848_1024 (size=253) 2024-11-14T20:30:11,074 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2, size=253 (253bytes) 2024-11-14T20:30:11,074 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2 2024-11-14T20:30:11,075 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2 after 1ms 2024-11-14T20:30:11,083 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:11,083 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2 took 9ms 2024-11-14T20:30:11,098 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2 so closing down 2024-11-14T20:30:11,098 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:11,113 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-14T20:30:11,117 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002-wal-2.temp 2024-11-14T20:30:11,121 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:11,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741849_1025 (size=253) 2024-11-14T20:30:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741849_1025 (size=253) 2024-11-14T20:30:11,202 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 4 ms) 2024-11-14T20:30:11,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741849_1025 (size=253) 2024-11-14T20:30:11,217 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002 2024-11-14T20:30:11,217 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-11-14T20:30:11,218 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2, journal: Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2, size=253 (253bytes) at 1731616211074Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2 so closing down at 1731616211098 (+24 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002-wal-2.temp at 1731616211117 (+19 ms)3 split writer threads finished at 1731616211121 (+4 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 4 ms) at 1731616211202 (+81 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002 at 1731616211217 (+15 ms)Processed 1 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1731616211217 2024-11-14T20:30:11,252 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1, size=320 (320bytes) 2024-11-14T20:30:11,252 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1 2024-11-14T20:30:11,263 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1 after 9ms 2024-11-14T20:30:11,283 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:11,283 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1 took 31ms 2024-11-14T20:30:11,303 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1 so closing down 2024-11-14T20:30:11,303 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:11,311 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-14T20:30:11,312 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000001-wal-1.temp 2024-11-14T20:30:11,313 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:11,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741850_1026 (size=320) 2024-11-14T20:30:11,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741850_1026 (size=320) 2024-11-14T20:30:11,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741850_1026 (size=320) 2024-11-14T20:30:11,345 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:11,351 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:11,353 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002, length=253 2024-11-14T20:30:11,356 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002 2024-11-14T20:30:11,356 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-11-14T20:30:11,356 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1, journal: Splitting hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1, size=320 (320bytes) at 1731616211252Finishing writing output for hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1 so closing down at 1731616211303 (+51 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000001-wal-1.temp at 1731616211312 (+9 ms)3 split writer threads finished at 1731616211313 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731616211345 (+32 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002 at 1731616211356 (+11 ms)Processed 2 edits across 1 Regions in 60 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1731616211356 2024-11-14T20:30:11,356 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:11,359 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:11,380 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal.1731616211360, exclude list is [], retry=0 2024-11-14T20:30:11,384 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:11,384 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:11,385 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:11,388 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal.1731616211360 2024-11-14T20:30:11,389 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:11,389 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 91e9249eb41850067c0d30835a773a0c, NAME => 'testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:11,389 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:11,389 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,389 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,395 INFO [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,398 INFO [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 91e9249eb41850067c0d30835a773a0c columnFamilyName a 2024-11-14T20:30:11,399 DEBUG [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:11,401 INFO [StoreOpener-91e9249eb41850067c0d30835a773a0c-1 {}] regionserver.HStore(327): Store=91e9249eb41850067c0d30835a773a0c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:11,401 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,402 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,405 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,406 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002 2024-11-14T20:30:11,413 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:11,416 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002 2024-11-14T20:30:11,416 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 91e9249eb41850067c0d30835a773a0c 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-14T20:30:11,441 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/.tmp/a/e6e85764e906485f897017fea3ca2b6f is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731616210915/Put/seqid=0 2024-11-14T20:30:11,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741852_1028 (size=5170) 2024-11-14T20:30:11,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741852_1028 (size=5170) 2024-11-14T20:30:11,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741852_1028 (size=5170) 2024-11-14T20:30:11,524 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/.tmp/a/e6e85764e906485f897017fea3ca2b6f 2024-11-14T20:30:11,570 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/.tmp/a/e6e85764e906485f897017fea3ca2b6f as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/a/e6e85764e906485f897017fea3ca2b6f 2024-11-14T20:30:11,617 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/a/e6e85764e906485f897017fea3ca2b6f, entries=2, sequenceid=2, filesize=5.0 K 2024-11-14T20:30:11,617 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 91e9249eb41850067c0d30835a773a0c in 201ms, sequenceid=2, compaction requested=false; wal=null 2024-11-14T20:30:11,627 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/0000000000000000002 2024-11-14T20:30:11,628 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,628 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 91e9249eb41850067c0d30835a773a0c 2024-11-14T20:30:11,664 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/91e9249eb41850067c0d30835a773a0c/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-14T20:30:11,668 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 91e9249eb41850067c0d30835a773a0c; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64202386, jitterRate=-0.04330989718437195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:11,669 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 91e9249eb41850067c0d30835a773a0c: Writing region info on filesystem at 1731616211389Initializing all the Stores at 1731616211391 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616211391Obtaining lock to block concurrent updates at 1731616211416 (+25 ms)Preparing flush snapshotting stores in 91e9249eb41850067c0d30835a773a0c at 1731616211416Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731616211416Flushing stores of testReplayEditsWrittenIntoWAL,,1731616210553.91e9249eb41850067c0d30835a773a0c. at 1731616211416Flushing 91e9249eb41850067c0d30835a773a0c/a: creating writer at 1731616211416Flushing 91e9249eb41850067c0d30835a773a0c/a: appending metadata at 1731616211439 (+23 ms)Flushing 91e9249eb41850067c0d30835a773a0c/a: closing flushed file at 1731616211439Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4aa504b0: reopening flushed file at 1731616211567 (+128 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 91e9249eb41850067c0d30835a773a0c in 201ms, sequenceid=2, compaction requested=false; wal=null at 1731616211617 (+50 ms)Cleaning up temporary data from old regions at 1731616211628 (+11 ms)Region opened successfully at 1731616211669 (+41 ms) 2024-11-14T20:30:11,713 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=384 (was 373) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:42250 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48124 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:57862 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48190 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:57766 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:39470 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=573 (was 573), ProcessCount=11 (was 11), AvailableMemoryMB=3731 (was 3677) - AvailableMemoryMB LEAK? - 2024-11-14T20:30:11,739 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=573, ProcessCount=11, AvailableMemoryMB=3730 2024-11-14T20:30:11,789 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:11,798 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:11,801 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:11,840 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-10720344, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-10720344, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:11,872 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-10720344/hregion-10720344.1731616211840, exclude list is [], retry=0 2024-11-14T20:30:11,893 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:11,905 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:11,931 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:11,961 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-10720344/hregion-10720344.1731616211840 2024-11-14T20:30:11,965 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:11,965 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ec4246877fee2607d32655bac38217ca, NAME => 'testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:12,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741854_1030 (size=64) 2024-11-14T20:30:12,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741854_1030 (size=64) 2024-11-14T20:30:12,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741854_1030 (size=64) 2024-11-14T20:30:12,017 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:12,022 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,027 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4246877fee2607d32655bac38217ca columnFamilyName a 2024-11-14T20:30:12,028 DEBUG [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:12,028 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(327): Store=ec4246877fee2607d32655bac38217ca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:12,028 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,030 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4246877fee2607d32655bac38217ca columnFamilyName b 2024-11-14T20:30:12,030 DEBUG [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:12,035 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(327): Store=ec4246877fee2607d32655bac38217ca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:12,035 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,039 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4246877fee2607d32655bac38217ca columnFamilyName c 2024-11-14T20:30:12,039 DEBUG [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:12,045 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(327): Store=ec4246877fee2607d32655bac38217ca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:12,045 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,047 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,048 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,057 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,057 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,058 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:12,060 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:12,069 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:12,069 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec4246877fee2607d32655bac38217ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67430012, jitterRate=0.004785478115081787}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:12,070 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec4246877fee2607d32655bac38217ca: Writing region info on filesystem at 1731616212017Initializing all the Stores at 1731616212020 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616212020Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616212021 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616212021Cleaning up temporary data from old regions at 1731616212057 (+36 ms)Region opened successfully at 1731616212070 (+13 ms) 2024-11-14T20:30:12,070 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec4246877fee2607d32655bac38217ca, disabling compactions & flushes 2024-11-14T20:30:12,070 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:12,070 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:12,070 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. after waiting 0 ms 2024-11-14T20:30:12,071 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:12,073 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:12,073 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec4246877fee2607d32655bac38217ca: Waiting for close lock at 1731616212070Disabling compacts and flushes for region at 1731616212070Disabling writes for close at 1731616212070Writing region close event to WAL at 1731616212073 (+3 ms)Closed at 1731616212073 2024-11-14T20:30:12,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741853_1029 (size=95) 2024-11-14T20:30:12,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741853_1029 (size=95) 2024-11-14T20:30:12,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741853_1029 (size=95) 2024-11-14T20:30:12,121 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:12,121 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-10720344:(num 1731616211840) 2024-11-14T20:30:12,122 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:12,141 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:12,157 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141, exclude list is [], retry=0 2024-11-14T20:30:12,169 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:12,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:12,170 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:12,208 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 2024-11-14T20:30:12,209 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:12,603 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141, size=0 (0bytes) 2024-11-14T20:30:12,603 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 might be still open, length is 0 2024-11-14T20:30:12,603 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 2024-11-14T20:30:12,605 WARN [IPC Server handler 0 on default port 43677 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-14T20:30:12,606 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 after 3ms 2024-11-14T20:30:12,966 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T20:30:13,012 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T20:30:13,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-14T20:30:13,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:13,836 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T20:30:13,836 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-14T20:30:13,837 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-14T20:30:13,837 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:13,838 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-14T20:30:13,838 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:13,965 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:42286 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41701:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42286 dst: /127.0.0.1:41701 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41701 remote=/127.0.0.1:42286]. Total timeout mills is 60000, 58570 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:13,967 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48228 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41951:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48228 dst: /127.0.0.1:41951 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:13,967 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:57892 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57892 dst: /127.0.0.1:37797 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:13,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741855_1032 (size=263633) 2024-11-14T20:30:13,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741855_1032 (size=263633) 2024-11-14T20:30:13,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741855_1032 (size=263633) 2024-11-14T20:30:16,607 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 after 4003ms 2024-11-14T20:30:16,612 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:16,614 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 took 4011ms 2024-11-14T20:30:16,626 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731616212141.temp 2024-11-14T20:30:16,636 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000000001-wal.1731616212141.temp 2024-11-14T20:30:16,806 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141; continuing. 2024-11-14T20:30:16,806 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 so closing down 2024-11-14T20:30:16,807 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:16,807 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:16,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741856_1033 (size=263641) 2024-11-14T20:30:16,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741856_1033 (size=263641) 2024-11-14T20:30:16,822 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000000001-wal.1731616212141.temp (wrote 3002 edits, skipped 0 edits in 72 ms) 2024-11-14T20:30:16,824 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000000001-wal.1731616212141.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000003002 2024-11-14T20:30:16,828 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 212 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T20:30:16,829 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141, journal: Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141, size=0 (0bytes) at 1731616212603Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000000001-wal.1731616212141.temp at 1731616216636 (+4033 ms)Split 1024 edits, skipped 0 edits. at 1731616216742 (+106 ms)Split 2048 edits, skipped 0 edits. at 1731616216770 (+28 ms)Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 so closing down at 1731616216806 (+36 ms)3 split writer threads finished at 1731616216807 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000000001-wal.1731616212141.temp (wrote 3002 edits, skipped 0 edits in 72 ms) at 1731616216822 (+15 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000000001-wal.1731616212141.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000003002 at 1731616216824 (+2 ms)Processed 3002 edits across 1 Regions in 212 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141, size=0, length=0, corrupted=false, cancelled=false at 1731616216829 (+5 ms) 2024-11-14T20:30:16,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741856_1033 (size=263641) 2024-11-14T20:30:16,834 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616212141 2024-11-14T20:30:16,841 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000003002 2024-11-14T20:30:16,841 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:16,846 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:16,860 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616216846, exclude list is [], retry=0 2024-11-14T20:30:16,864 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:16,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:16,865 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:16,871 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616216846 2024-11-14T20:30:16,871 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:16,871 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:16,877 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:16,881 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4246877fee2607d32655bac38217ca columnFamilyName a 2024-11-14T20:30:16,881 DEBUG [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:16,882 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(327): Store=ec4246877fee2607d32655bac38217ca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:16,882 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:16,884 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4246877fee2607d32655bac38217ca columnFamilyName b 2024-11-14T20:30:16,884 DEBUG [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:16,884 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(327): Store=ec4246877fee2607d32655bac38217ca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:16,884 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:16,887 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec4246877fee2607d32655bac38217ca columnFamilyName c 2024-11-14T20:30:16,887 DEBUG [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:16,887 INFO [StoreOpener-ec4246877fee2607d32655bac38217ca-1 {}] regionserver.HStore(327): Store=ec4246877fee2607d32655bac38217ca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:16,888 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:16,889 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:16,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:16,893 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000003002 2024-11-14T20:30:16,898 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:16,969 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T20:30:17,473 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4246877fee2607d32655bac38217ca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T20:30:17,535 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/a/5d1065678a42483b87ee57ad9d3a8ce3 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731616212220/Put/seqid=0 2024-11-14T20:30:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741858_1035 (size=50463) 2024-11-14T20:30:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741858_1035 (size=50463) 2024-11-14T20:30:17,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741858_1035 (size=50463) 2024-11-14T20:30:17,957 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/a/5d1065678a42483b87ee57ad9d3a8ce3 2024-11-14T20:30:17,968 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/a/5d1065678a42483b87ee57ad9d3a8ce3 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/a/5d1065678a42483b87ee57ad9d3a8ce3 2024-11-14T20:30:17,989 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/a/5d1065678a42483b87ee57ad9d3a8ce3, entries=754, sequenceid=754, filesize=49.3 K 2024-11-14T20:30:17,990 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for ec4246877fee2607d32655bac38217ca in 516ms, sequenceid=754, compaction requested=false; wal=null 2024-11-14T20:30:18,008 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T20:30:18,008 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4246877fee2607d32655bac38217ca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T20:30:18,018 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/a/1a9b1b7e378841ca824a904892b9a3a7 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731616212307/Put/seqid=0 2024-11-14T20:30:18,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741859_1036 (size=20072) 2024-11-14T20:30:18,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741859_1036 (size=20072) 2024-11-14T20:30:18,034 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/a/1a9b1b7e378841ca824a904892b9a3a7 2024-11-14T20:30:18,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741859_1036 (size=20072) 2024-11-14T20:30:18,080 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/b/6b606de774dd4671886ae36eb5f9c2c0 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731616212365/Put/seqid=0 2024-11-14T20:30:18,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741860_1037 (size=35835) 2024-11-14T20:30:18,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741860_1037 (size=35835) 2024-11-14T20:30:18,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741860_1037 (size=35835) 2024-11-14T20:30:18,105 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/b/6b606de774dd4671886ae36eb5f9c2c0 2024-11-14T20:30:18,122 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/a/1a9b1b7e378841ca824a904892b9a3a7 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/a/1a9b1b7e378841ca824a904892b9a3a7 2024-11-14T20:30:18,137 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/a/1a9b1b7e378841ca824a904892b9a3a7, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-14T20:30:18,139 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/b/6b606de774dd4671886ae36eb5f9c2c0 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/b/6b606de774dd4671886ae36eb5f9c2c0 2024-11-14T20:30:18,148 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/b/6b606de774dd4671886ae36eb5f9c2c0, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-14T20:30:18,149 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ec4246877fee2607d32655bac38217ca in 140ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-14T20:30:18,170 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T20:30:18,170 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4246877fee2607d32655bac38217ca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-14T20:30:18,202 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/b/c92ae0696e764951875e3bcd477e5098 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731616212411/Put/seqid=0 2024-11-14T20:30:18,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741861_1038 (size=35082) 2024-11-14T20:30:18,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741861_1038 (size=35082) 2024-11-14T20:30:18,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741861_1038 (size=35082) 2024-11-14T20:30:18,228 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/b/c92ae0696e764951875e3bcd477e5098 2024-11-14T20:30:18,288 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/c/a792e397ccc7405ba65f7bf7bdf9698d is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731616212476/Put/seqid=0 2024-11-14T20:30:18,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741862_1039 (size=20825) 2024-11-14T20:30:18,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741862_1039 (size=20825) 2024-11-14T20:30:18,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741862_1039 (size=20825) 2024-11-14T20:30:18,711 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/c/a792e397ccc7405ba65f7bf7bdf9698d 2024-11-14T20:30:18,719 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/b/c92ae0696e764951875e3bcd477e5098 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/b/c92ae0696e764951875e3bcd477e5098 2024-11-14T20:30:18,727 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/b/c92ae0696e764951875e3bcd477e5098, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-14T20:30:18,728 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/c/a792e397ccc7405ba65f7bf7bdf9698d as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/c/a792e397ccc7405ba65f7bf7bdf9698d 2024-11-14T20:30:18,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/c/a792e397ccc7405ba65f7bf7bdf9698d, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-14T20:30:18,735 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for ec4246877fee2607d32655bac38217ca in 565ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-14T20:30:18,748 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731616212533/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:18,752 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000003002 2024-11-14T20:30:18,752 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-14T20:30:18,753 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec4246877fee2607d32655bac38217ca 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-14T20:30:18,763 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/c/57d2640662bf4f97b0a6b7bf54920bf7 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731616212487/Put/seqid=0 2024-11-14T20:30:18,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741863_1040 (size=50301) 2024-11-14T20:30:18,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741863_1040 (size=50301) 2024-11-14T20:30:18,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741863_1040 (size=50301) 2024-11-14T20:30:18,794 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/c/57d2640662bf4f97b0a6b7bf54920bf7 2024-11-14T20:30:18,802 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57d2640662bf4f97b0a6b7bf54920bf7 2024-11-14T20:30:18,804 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/.tmp/c/57d2640662bf4f97b0a6b7bf54920bf7 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/c/57d2640662bf4f97b0a6b7bf54920bf7 2024-11-14T20:30:18,812 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 57d2640662bf4f97b0a6b7bf54920bf7 2024-11-14T20:30:18,812 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/c/57d2640662bf4f97b0a6b7bf54920bf7, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-14T20:30:18,813 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for ec4246877fee2607d32655bac38217ca in 60ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-14T20:30:18,814 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/0000000000000003002 2024-11-14T20:30:18,815 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:18,815 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:18,816 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-14T20:30:18,821 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec4246877fee2607d32655bac38217ca 2024-11-14T20:30:18,833 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenIntoWAL/ec4246877fee2607d32655bac38217ca/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-14T20:30:18,834 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec4246877fee2607d32655bac38217ca; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65611554, jitterRate=-0.022311657667160034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-14T20:30:18,835 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec4246877fee2607d32655bac38217ca: Writing region info on filesystem at 1731616216872Initializing all the Stores at 1731616216873 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616216873Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616216876 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616216876Cleaning up temporary data from old regions at 1731616218815 (+1939 ms)Region opened successfully at 1731616218834 (+19 ms) 2024-11-14T20:30:18,914 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec4246877fee2607d32655bac38217ca, disabling compactions & flushes 2024-11-14T20:30:18,914 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:18,915 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:18,915 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. after waiting 0 ms 2024-11-14T20:30:18,915 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:18,925 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731616211789.ec4246877fee2607d32655bac38217ca. 2024-11-14T20:30:18,925 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec4246877fee2607d32655bac38217ca: Waiting for close lock at 1731616218914Disabling compacts and flushes for region at 1731616218914Disabling writes for close at 1731616218915 (+1 ms)Writing region close event to WAL at 1731616218925 (+10 ms)Closed at 1731616218925 2024-11-14T20:30:18,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741857_1034 (size=95) 2024-11-14T20:30:18,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741857_1034 (size=95) 2024-11-14T20:30:18,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741857_1034 (size=95) 2024-11-14T20:30:18,941 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:18,941 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731616216846) 2024-11-14T20:30:18,959 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=395 (was 384) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_414936404_22 at /127.0.0.1:53796 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_414936404_22 at /127.0.0.1:52162 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43677 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43777 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_414936404_22 at /127.0.0.1:53772 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43777 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_414936404_22 at /127.0.0.1:52770 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38913 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43677 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:38913 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=859 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 573) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4570 (was 3730) - AvailableMemoryMB LEAK? - 2024-11-14T20:30:18,976 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=395, OpenFileDescriptor=859, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=4568 2024-11-14T20:30:18,999 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:19,002 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:19,003 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:19,007 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-21627444, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-21627444, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:19,028 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-21627444/hregion-21627444.1731616219007, exclude list is [], retry=0 2024-11-14T20:30:19,032 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:19,032 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:19,032 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:19,038 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-21627444/hregion-21627444.1731616219007 2024-11-14T20:30:19,038 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:19,038 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 0242ace55bce406fa627810a718f071a, NAME => 'test2727,,1731616219000.0242ace55bce406fa627810a718f071a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:19,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741865_1042 (size=43) 2024-11-14T20:30:19,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741865_1042 (size=43) 2024-11-14T20:30:19,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741865_1042 (size=43) 2024-11-14T20:30:19,062 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731616219000.0242ace55bce406fa627810a718f071a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:19,063 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,065 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0242ace55bce406fa627810a718f071a columnFamilyName a 2024-11-14T20:30:19,065 DEBUG [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:19,066 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(327): Store=0242ace55bce406fa627810a718f071a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:19,066 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,068 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0242ace55bce406fa627810a718f071a columnFamilyName b 2024-11-14T20:30:19,068 DEBUG [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:19,068 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(327): Store=0242ace55bce406fa627810a718f071a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:19,068 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,070 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0242ace55bce406fa627810a718f071a columnFamilyName c 2024-11-14T20:30:19,070 DEBUG [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:19,070 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(327): Store=0242ace55bce406fa627810a718f071a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:19,071 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,071 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,072 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,074 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,074 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,074 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:19,085 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,097 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:19,098 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0242ace55bce406fa627810a718f071a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68813950, jitterRate=0.025407761335372925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0242ace55bce406fa627810a718f071a: Writing region info on filesystem at 1731616219062Initializing all the Stores at 1731616219063 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616219063Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616219063Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616219063Cleaning up temporary data from old regions at 1731616219074 (+11 ms)Region opened successfully at 1731616219099 (+25 ms) 2024-11-14T20:30:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0242ace55bce406fa627810a718f071a, disabling compactions & flushes 2024-11-14T20:30:19,099 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731616219000.0242ace55bce406fa627810a718f071a. after waiting 0 ms 2024-11-14T20:30:19,099 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:19,101 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:19,101 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0242ace55bce406fa627810a718f071a: Waiting for close lock at 1731616219099Disabling compacts and flushes for region at 1731616219099Disabling writes for close at 1731616219099Writing region close event to WAL at 1731616219100 (+1 ms)Closed at 1731616219101 (+1 ms) 2024-11-14T20:30:19,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741864_1041 (size=95) 2024-11-14T20:30:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741864_1041 (size=95) 2024-11-14T20:30:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741864_1041 (size=95) 2024-11-14T20:30:19,111 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:19,111 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-21627444:(num 1731616219007) 2024-11-14T20:30:19,112 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:19,113 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:19,127 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114, exclude list is [], retry=0 2024-11-14T20:30:19,130 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:19,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:19,131 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:19,141 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114 2024-11-14T20:30:19,141 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:19,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741866_1043 (size=263359) 2024-11-14T20:30:19,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741866_1043 (size=263359) 2024-11-14T20:30:19,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741866_1043 (size=263359) 2024-11-14T20:30:19,370 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114, size=257.2 K (263359bytes) 2024-11-14T20:30:19,370 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114 2024-11-14T20:30:19,371 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114 after 0ms 2024-11-14T20:30:19,377 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:19,379 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114 took 9ms 2024-11-14T20:30:19,386 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731616219114.temp 2024-11-14T20:30:19,392 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000000001-wal.1731616219114.temp 2024-11-14T20:30:19,452 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114 so closing down 2024-11-14T20:30:19,452 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:19,453 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:19,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741867_1044 (size=263359) 2024-11-14T20:30:19,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741867_1044 (size=263359) 2024-11-14T20:30:19,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741867_1044 (size=263359) 2024-11-14T20:30:19,480 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000000001-wal.1731616219114.temp (wrote 3000 edits, skipped 0 edits in 51 ms) 2024-11-14T20:30:19,482 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000000001-wal.1731616219114.temp to hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003000 2024-11-14T20:30:19,483 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-11-14T20:30:19,483 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114, journal: Splitting hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114, size=257.2 K (263359bytes) at 1731616219370Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000000001-wal.1731616219114.temp at 1731616219392 (+22 ms)Split 1024 edits, skipped 0 edits. at 1731616219407 (+15 ms)Split 2048 edits, skipped 0 edits. at 1731616219432 (+25 ms)Finishing writing output for hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114 so closing down at 1731616219452 (+20 ms)3 split writer threads finished at 1731616219453 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000000001-wal.1731616219114.temp (wrote 3000 edits, skipped 0 edits in 51 ms) at 1731616219480 (+27 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000000001-wal.1731616219114.temp to hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003000 at 1731616219482 (+2 ms)Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1731616219483 (+1 ms) 2024-11-14T20:30:19,485 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219114 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616219114 2024-11-14T20:30:19,486 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003000 2024-11-14T20:30:19,486 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:19,488 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:19,511 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489, exclude list is [], retry=0 2024-11-14T20:30:19,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:19,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:19,519 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:19,529 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489 2024-11-14T20:30:19,532 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:19,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741868_1045 (size=263486) 2024-11-14T20:30:19,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741868_1045 (size=263486) 2024-11-14T20:30:19,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741868_1045 (size=263486) 2024-11-14T20:30:19,814 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489, size=257.3 K (263486bytes) 2024-11-14T20:30:19,815 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489 2024-11-14T20:30:19,817 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489 after 2ms 2024-11-14T20:30:19,821 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:19,828 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489 took 15ms 2024-11-14T20:30:19,836 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731616219489.temp 2024-11-14T20:30:19,838 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003001-wal.1731616219489.temp 2024-11-14T20:30:19,921 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489 so closing down 2024-11-14T20:30:19,921 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:19,921 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:19,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741869_1046 (size=263486) 2024-11-14T20:30:19,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741869_1046 (size=263486) 2024-11-14T20:30:19,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741869_1046 (size=263486) 2024-11-14T20:30:19,926 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003001-wal.1731616219489.temp (wrote 3000 edits, skipped 0 edits in 50 ms) 2024-11-14T20:30:19,929 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003001-wal.1731616219489.temp to hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000006000 2024-11-14T20:30:19,929 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 97 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-11-14T20:30:19,929 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489, journal: Splitting hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489, size=257.3 K (263486bytes) at 1731616219814Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003001-wal.1731616219489.temp at 1731616219838 (+24 ms)Split 1024 edits, skipped 0 edits. at 1731616219861 (+23 ms)Split 2048 edits, skipped 0 edits. at 1731616219893 (+32 ms)Finishing writing output for hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489 so closing down at 1731616219921 (+28 ms)3 split writer threads finished at 1731616219921Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003001-wal.1731616219489.temp (wrote 3000 edits, skipped 0 edits in 50 ms) at 1731616219926 (+5 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003001-wal.1731616219489.temp to hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000006000 at 1731616219929 (+3 ms)Processed 3000 edits across 1 Regions in 97 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1731616219929 2024-11-14T20:30:19,933 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219489 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616219489 2024-11-14T20:30:19,935 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000006000 2024-11-14T20:30:19,935 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:19,938 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/test2727-manual,16010,1731616218998, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:19,965 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219938, exclude list is [], retry=0 2024-11-14T20:30:19,972 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:19,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:19,973 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:19,988 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731616218998/wal.1731616219938 2024-11-14T20:30:19,989 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:19,989 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 0242ace55bce406fa627810a718f071a, NAME => 'test2727,,1731616219000.0242ace55bce406fa627810a718f071a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:19,989 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731616219000.0242ace55bce406fa627810a718f071a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:19,990 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,990 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,993 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:19,997 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0242ace55bce406fa627810a718f071a columnFamilyName a 2024-11-14T20:30:19,998 DEBUG [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:19,998 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(327): Store=0242ace55bce406fa627810a718f071a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:19,999 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,002 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0242ace55bce406fa627810a718f071a columnFamilyName b 2024-11-14T20:30:20,002 DEBUG [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:20,003 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(327): Store=0242ace55bce406fa627810a718f071a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:20,003 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,005 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0242ace55bce406fa627810a718f071a columnFamilyName c 2024-11-14T20:30:20,006 DEBUG [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:20,006 INFO [StoreOpener-0242ace55bce406fa627810a718f071a-1 {}] regionserver.HStore(327): Store=0242ace55bce406fa627810a718f071a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:20,007 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,008 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,012 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,013 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003000 2024-11-14T20:30:20,017 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:20,113 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003000 2024-11-14T20:30:20,117 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000006000 2024-11-14T20:30:20,123 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:20,203 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000006000 2024-11-14T20:30:20,204 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 0242ace55bce406fa627810a718f071a 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-14T20:30:20,240 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/a/4c4168e6d35d4c3998263eb56678dd20 is 41, key is test2727/a:100/1731616219538/Put/seqid=0 2024-11-14T20:30:20,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741871_1048 (size=84227) 2024-11-14T20:30:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741871_1048 (size=84227) 2024-11-14T20:30:20,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741871_1048 (size=84227) 2024-11-14T20:30:20,262 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/a/4c4168e6d35d4c3998263eb56678dd20 2024-11-14T20:30:20,295 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/b/120f202a26a442a5a84f3ea450315871 is 41, key is test2727/b:100/1731616219610/Put/seqid=0 2024-11-14T20:30:20,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741872_1049 (size=84609) 2024-11-14T20:30:20,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741872_1049 (size=84609) 2024-11-14T20:30:20,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741872_1049 (size=84609) 2024-11-14T20:30:20,312 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/b/120f202a26a442a5a84f3ea450315871 2024-11-14T20:30:20,365 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/c/c642183b606148b5971bf96d453f76a1 is 41, key is test2727/c:100/1731616219683/Put/seqid=0 2024-11-14T20:30:20,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741873_1050 (size=84609) 2024-11-14T20:30:20,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741873_1050 (size=84609) 2024-11-14T20:30:20,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741873_1050 (size=84609) 2024-11-14T20:30:20,389 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/c/c642183b606148b5971bf96d453f76a1 2024-11-14T20:30:20,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/a/4c4168e6d35d4c3998263eb56678dd20 as hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/a/4c4168e6d35d4c3998263eb56678dd20 2024-11-14T20:30:20,406 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/a/4c4168e6d35d4c3998263eb56678dd20, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-14T20:30:20,408 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/b/120f202a26a442a5a84f3ea450315871 as hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/b/120f202a26a442a5a84f3ea450315871 2024-11-14T20:30:20,428 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/b/120f202a26a442a5a84f3ea450315871, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T20:30:20,434 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/.tmp/c/c642183b606148b5971bf96d453f76a1 as hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/c/c642183b606148b5971bf96d453f76a1 2024-11-14T20:30:20,452 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/c/c642183b606148b5971bf96d453f76a1, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-14T20:30:20,453 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0242ace55bce406fa627810a718f071a in 249ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-14T20:30:20,455 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000003000 2024-11-14T20:30:20,456 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/0000000000000006000 2024-11-14T20:30:20,457 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,457 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,458 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:20,461 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 0242ace55bce406fa627810a718f071a 2024-11-14T20:30:20,468 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/test2727/0242ace55bce406fa627810a718f071a/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-14T20:30:20,470 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 0242ace55bce406fa627810a718f071a; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65405493, jitterRate=-0.025382205843925476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:20,471 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 0242ace55bce406fa627810a718f071a: Writing region info on filesystem at 1731616219990Initializing all the Stores at 1731616219992 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616219992Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616219992Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616219992Obtaining lock to block concurrent updates at 1731616220204 (+212 ms)Preparing flush snapshotting stores in 0242ace55bce406fa627810a718f071a at 1731616220204Finished memstore snapshotting test2727,,1731616219000.0242ace55bce406fa627810a718f071a., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731616220204Flushing stores of test2727,,1731616219000.0242ace55bce406fa627810a718f071a. at 1731616220204Flushing 0242ace55bce406fa627810a718f071a/a: creating writer at 1731616220204Flushing 0242ace55bce406fa627810a718f071a/a: appending metadata at 1731616220238 (+34 ms)Flushing 0242ace55bce406fa627810a718f071a/a: closing flushed file at 1731616220238Flushing 0242ace55bce406fa627810a718f071a/b: creating writer at 1731616220269 (+31 ms)Flushing 0242ace55bce406fa627810a718f071a/b: appending metadata at 1731616220293 (+24 ms)Flushing 0242ace55bce406fa627810a718f071a/b: closing flushed file at 1731616220293Flushing 0242ace55bce406fa627810a718f071a/c: creating writer at 1731616220331 (+38 ms)Flushing 0242ace55bce406fa627810a718f071a/c: appending metadata at 1731616220364 (+33 ms)Flushing 0242ace55bce406fa627810a718f071a/c: closing flushed file at 1731616220364Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37ea366d: reopening flushed file at 1731616220397 (+33 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42936fce: reopening flushed file at 1731616220407 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23c3c0f2: reopening flushed file at 1731616220429 (+22 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 0242ace55bce406fa627810a718f071a in 249ms, sequenceid=6000, compaction requested=false; wal=null at 1731616220453 (+24 ms)Cleaning up temporary data from old regions at 1731616220457 (+4 ms)Region opened successfully at 1731616220471 (+14 ms) 2024-11-14T20:30:20,474 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-14T20:30:20,474 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 0242ace55bce406fa627810a718f071a, disabling compactions & flushes 2024-11-14T20:30:20,474 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:20,475 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:20,475 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731616219000.0242ace55bce406fa627810a718f071a. after waiting 0 ms 2024-11-14T20:30:20,475 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:20,487 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731616219000.0242ace55bce406fa627810a718f071a. 2024-11-14T20:30:20,487 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 0242ace55bce406fa627810a718f071a: Waiting for close lock at 1731616220474Disabling compacts and flushes for region at 1731616220474Disabling writes for close at 1731616220475 (+1 ms)Writing region close event to WAL at 1731616220487 (+12 ms)Closed at 1731616220487 2024-11-14T20:30:20,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741870_1047 (size=95) 2024-11-14T20:30:20,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741870_1047 (size=95) 2024-11-14T20:30:20,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741870_1047 (size=95) 2024-11-14T20:30:20,497 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:20,498 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731616219938) 2024-11-14T20:30:20,516 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=398 (was 395) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:53796 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:52162 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:52932 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=921 (was 859) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 690), ProcessCount=11 (was 11), AvailableMemoryMB=4393 (was 4568) 2024-11-14T20:30:20,534 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=398, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=4392 2024-11-14T20:30:20,567 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:20,576 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:20,578 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731616220578 2024-11-14T20:30:20,594 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578 2024-11-14T20:30:20,597 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:20,604 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 35f53d2902a714902409ef9fc464b8f6, NAME => 'testSequentialEditLogSeqNum,,1731616220568.35f53d2902a714902409ef9fc464b8f6.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:20,604 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731616220568.35f53d2902a714902409ef9fc464b8f6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:20,605 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,605 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,606 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6 doesn't exist for region: 35f53d2902a714902409ef9fc464b8f6 on table testSequentialEditLogSeqNum 2024-11-14T20:30:20,607 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 35f53d2902a714902409ef9fc464b8f6 on table testSequentialEditLogSeqNum 2024-11-14T20:30:20,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741875_1052 (size=62) 2024-11-14T20:30:20,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741875_1052 (size=62) 2024-11-14T20:30:20,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741875_1052 (size=62) 2024-11-14T20:30:20,648 INFO [StoreOpener-35f53d2902a714902409ef9fc464b8f6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,651 INFO [StoreOpener-35f53d2902a714902409ef9fc464b8f6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 35f53d2902a714902409ef9fc464b8f6 columnFamilyName a 2024-11-14T20:30:20,651 DEBUG [StoreOpener-35f53d2902a714902409ef9fc464b8f6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:20,652 INFO [StoreOpener-35f53d2902a714902409ef9fc464b8f6-1 {}] regionserver.HStore(327): Store=35f53d2902a714902409ef9fc464b8f6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:20,652 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,653 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,653 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,654 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,654 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,656 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 35f53d2902a714902409ef9fc464b8f6 2024-11-14T20:30:20,659 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:20,659 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 35f53d2902a714902409ef9fc464b8f6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62019193, jitterRate=-0.0758420079946518}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:20,661 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 35f53d2902a714902409ef9fc464b8f6: Writing region info on filesystem at 1731616220605Initializing all the Stores at 1731616220644 (+39 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616220644Cleaning up temporary data from old regions at 1731616220654 (+10 ms)Region opened successfully at 1731616220661 (+7 ms) 2024-11-14T20:30:20,680 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 35f53d2902a714902409ef9fc464b8f6 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-14T20:30:20,706 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/.tmp/a/a429b1ce82044aea9ebadd92de50171d is 81, key is testSequentialEditLogSeqNum/a:x0/1731616220661/Put/seqid=0 2024-11-14T20:30:20,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741876_1053 (size=5833) 2024-11-14T20:30:20,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741876_1053 (size=5833) 2024-11-14T20:30:20,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741876_1053 (size=5833) 2024-11-14T20:30:20,715 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/.tmp/a/a429b1ce82044aea9ebadd92de50171d 2024-11-14T20:30:20,723 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/.tmp/a/a429b1ce82044aea9ebadd92de50171d as hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/a/a429b1ce82044aea9ebadd92de50171d 2024-11-14T20:30:20,733 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/a/a429b1ce82044aea9ebadd92de50171d, entries=10, sequenceid=13, filesize=5.7 K 2024-11-14T20:30:20,735 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 35f53d2902a714902409ef9fc464b8f6 in 54ms, sequenceid=13, compaction requested=false 2024-11-14T20:30:20,735 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 35f53d2902a714902409ef9fc464b8f6: 2024-11-14T20:30:20,742 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T20:30:20,742 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T20:30:20,742 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T20:30:20,742 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T20:30:20,742 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-14T20:30:20,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741874_1051 (size=1844) 2024-11-14T20:30:20,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741874_1051 (size=1844) 2024-11-14T20:30:20,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741874_1051 (size=1844) 2024-11-14T20:30:20,771 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578, size=1.8 K (1844bytes) 2024-11-14T20:30:20,771 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578 2024-11-14T20:30:20,772 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578 after 0ms 2024-11-14T20:30:20,776 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:20,776 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578 took 5ms 2024-11-14T20:30:20,793 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578 so closing down 2024-11-14T20:30:20,793 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:20,808 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731616220578.temp 2024-11-14T20:30:20,813 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000003-wal.1731616220578.temp 2024-11-14T20:30:20,814 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:20,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741877_1054 (size=1477) 2024-11-14T20:30:20,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741877_1054 (size=1477) 2024-11-14T20:30:20,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741877_1054 (size=1477) 2024-11-14T20:30:20,864 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000003-wal.1731616220578.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:20,870 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000003-wal.1731616220578.temp to hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000020 2024-11-14T20:30:20,870 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 90 ms; skipped=2; WAL=hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578, size=1.8 K, length=1844, corrupted=false, cancelled=false 2024-11-14T20:30:20,870 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578, journal: Splitting hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578, size=1.8 K (1844bytes) at 1731616220771Finishing writing output for hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578 so closing down at 1731616220793 (+22 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000003-wal.1731616220578.temp at 1731616220813 (+20 ms)3 split writer threads finished at 1731616220814 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000003-wal.1731616220578.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731616220864 (+50 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000003-wal.1731616220578.temp to hdfs://localhost:43677/hbase/data/default/testSequentialEditLogSeqNum/35f53d2902a714902409ef9fc464b8f6/recovered.edits/0000000000000000020 at 1731616220870 (+6 ms)Processed 17 edits across 1 Regions in 90 ms; skipped=2; WAL=hdfs://localhost:43677/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731616220566/wal.1731616220578, size=1.8 K, length=1844, corrupted=false, cancelled=false at 1731616220870 2024-11-14T20:30:20,907 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=403 (was 398) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:53796 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:52162 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=690 (was 690), ProcessCount=11 (was 11), AvailableMemoryMB=4363 (was 4392) 2024-11-14T20:30:20,925 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=403, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=690, ProcessCount=11, AvailableMemoryMB=4360 2024-11-14T20:30:20,961 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:20,968 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:20,993 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:20,998 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-61776335, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-61776335, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:21,021 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-61776335/hregion-61776335.1731616220999, exclude list is [], retry=0 2024-11-14T20:30:21,026 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:21,026 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:21,028 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:21,037 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-61776335/hregion-61776335.1731616220999 2024-11-14T20:30:21,041 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:21,041 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => db92926d22002a42903150cc07bdb7f8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:21,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741879_1056 (size=70) 2024-11-14T20:30:21,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741879_1056 (size=70) 2024-11-14T20:30:21,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741879_1056 (size=70) 2024-11-14T20:30:21,069 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:21,080 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,083 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName a 2024-11-14T20:30:21,084 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:21,090 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:21,091 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,092 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName b 2024-11-14T20:30:21,093 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:21,100 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:21,100 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,102 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName c 2024-11-14T20:30:21,103 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:21,103 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:21,103 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,105 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,107 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,107 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,107 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:21,109 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,113 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:21,114 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db92926d22002a42903150cc07bdb7f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74781032, jitterRate=0.11432421207427979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:21,116 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db92926d22002a42903150cc07bdb7f8: Writing region info on filesystem at 1731616221069Initializing all the Stores at 1731616221077 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616221077Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616221079 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616221079Cleaning up temporary data from old regions at 1731616221107 (+28 ms)Region opened successfully at 1731616221116 (+9 ms) 2024-11-14T20:30:21,117 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing db92926d22002a42903150cc07bdb7f8, disabling compactions & flushes 2024-11-14T20:30:21,117 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:21,117 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:21,117 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. after waiting 0 ms 2024-11-14T20:30:21,117 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:21,121 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:21,121 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for db92926d22002a42903150cc07bdb7f8: Waiting for close lock at 1731616221117Disabling compacts and flushes for region at 1731616221117Disabling writes for close at 1731616221117Writing region close event to WAL at 1731616221121 (+4 ms)Closed at 1731616221121 2024-11-14T20:30:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741878_1055 (size=95) 2024-11-14T20:30:21,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741878_1055 (size=95) 2024-11-14T20:30:21,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741878_1055 (size=95) 2024-11-14T20:30:21,135 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:21,135 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-61776335:(num 1731616220999) 2024-11-14T20:30:21,135 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:21,138 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:21,161 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139, exclude list is [], retry=0 2024-11-14T20:30:21,165 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:21,165 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:21,166 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:21,185 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 2024-11-14T20:30:21,189 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:21,189 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => db92926d22002a42903150cc07bdb7f8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:21,190 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:21,190 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,190 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,192 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,193 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName a 2024-11-14T20:30:21,193 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:21,194 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:21,194 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,195 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName b 2024-11-14T20:30:21,195 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:21,196 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:21,196 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,197 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName c 2024-11-14T20:30:21,197 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:21,198 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:21,198 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,206 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,212 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:21,215 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:21,216 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db92926d22002a42903150cc07bdb7f8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69040390, jitterRate=0.028781980276107788}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:21,217 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db92926d22002a42903150cc07bdb7f8: Writing region info on filesystem at 1731616221190Initializing all the Stores at 1731616221191 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616221191Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616221191Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616221191Cleaning up temporary data from old regions at 1731616221211 (+20 ms)Region opened successfully at 1731616221217 (+6 ms) 2024-11-14T20:30:21,230 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731616221228/Put/seqid=0 2024-11-14T20:30:21,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741881_1058 (size=4826) 2024-11-14T20:30:21,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741881_1058 (size=4826) 2024-11-14T20:30:21,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741881_1058 (size=4826) 2024-11-14T20:30:21,246 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43677/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in db92926d22002a42903150cc07bdb7f8/a 2024-11-14T20:30:21,262 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-14T20:30:21,263 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T20:30:21,263 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for db92926d22002a42903150cc07bdb7f8: 2024-11-14T20:30:21,266 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/a/05343e17767d4543b7ea09e1ebe67194_SeqId_3_ 2024-11-14T20:30:21,267 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43677/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into db92926d22002a42903150cc07bdb7f8/a as hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/a/05343e17767d4543b7ea09e1ebe67194_SeqId_3_ - updating store file list. 2024-11-14T20:30:21,278 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 05343e17767d4543b7ea09e1ebe67194_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:21,279 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/a/05343e17767d4543b7ea09e1ebe67194_SeqId_3_ into db92926d22002a42903150cc07bdb7f8/a 2024-11-14T20:30:21,279 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43677/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into db92926d22002a42903150cc07bdb7f8/a (new location: hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/a/05343e17767d4543b7ea09e1ebe67194_SeqId_3_) 2024-11-14T20:30:21,337 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139, size=0 (0bytes) 2024-11-14T20:30:21,337 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 might be still open, length is 0 2024-11-14T20:30:21,337 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 2024-11-14T20:30:21,337 WARN [IPC Server handler 1 on default port 43677 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-14T20:30:21,338 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 after 1ms 2024-11-14T20:30:22,952 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:52342 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41951:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52342 dst: /127.0.0.1:41951 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41951 remote=/127.0.0.1:52342]. Total timeout mills is 60000, 58339 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:22,952 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:53994 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53994 dst: /127.0.0.1:37797 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:22,952 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:53012 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41701:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53012 dst: /127.0.0.1:41701 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:22,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741880_1059 (size=474) 2024-11-14T20:30:22,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741880_1059 (size=474) 2024-11-14T20:30:22,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741880_1059 (size=474) 2024-11-14T20:30:23,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-14T20:30:23,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:23,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-14T20:30:23,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:25,339 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 after 4002ms 2024-11-14T20:30:25,347 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:25,347 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 took 4010ms 2024-11-14T20:30:25,366 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139; continuing. 2024-11-14T20:30:25,366 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 so closing down 2024-11-14T20:30:25,367 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:25,379 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731616221139.temp 2024-11-14T20:30:25,384 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005-wal.1731616221139.temp 2024-11-14T20:30:25,384 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:25,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741882_1060 (size=259) 2024-11-14T20:30:25,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741882_1060 (size=259) 2024-11-14T20:30:25,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741882_1060 (size=259) 2024-11-14T20:30:25,403 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005-wal.1731616221139.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:25,405 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005-wal.1731616221139.temp to hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005 2024-11-14T20:30:25,406 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T20:30:25,406 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139, journal: Splitting hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139, size=0 (0bytes) at 1731616221337Finishing writing output for hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 so closing down at 1731616225366 (+4029 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005-wal.1731616221139.temp at 1731616225384 (+18 ms)3 split writer threads finished at 1731616225385 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005-wal.1731616221139.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731616225403 (+18 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005-wal.1731616221139.temp to hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005 at 1731616225406 (+3 ms)Processed 2 edits across 1 Regions in 46 ms; skipped=1; WAL=hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139, size=0, length=0, corrupted=false, cancelled=false at 1731616225406 2024-11-14T20:30:25,408 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616221139 2024-11-14T20:30:25,409 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005 2024-11-14T20:30:25,409 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:25,411 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:25,433 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616225411, exclude list is [], retry=0 2024-11-14T20:30:25,437 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:25,438 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:25,438 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:25,441 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616225411 2024-11-14T20:30:25,441 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:25,441 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => db92926d22002a42903150cc07bdb7f8, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:25,442 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:25,442 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,442 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,445 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,447 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName a 2024-11-14T20:30:25,447 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:25,457 DEBUG [StoreFileOpener-db92926d22002a42903150cc07bdb7f8-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 05343e17767d4543b7ea09e1ebe67194_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:25,457 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/a/05343e17767d4543b7ea09e1ebe67194_SeqId_3_ 2024-11-14T20:30:25,457 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:25,457 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,458 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName b 2024-11-14T20:30:25,458 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:25,460 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:25,461 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,462 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db92926d22002a42903150cc07bdb7f8 columnFamilyName c 2024-11-14T20:30:25,462 DEBUG [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:25,462 INFO [StoreOpener-db92926d22002a42903150cc07bdb7f8-1 {}] regionserver.HStore(327): Store=db92926d22002a42903150cc07bdb7f8/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:25,463 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,465 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,466 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005 2024-11-14T20:30:25,468 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:25,470 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005 2024-11-14T20:30:25,470 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing db92926d22002a42903150cc07bdb7f8 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-14T20:30:25,491 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/.tmp/a/3d9d5e6c938344b687da96f0b38afe26 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731616221289/Put/seqid=0 2024-11-14T20:30:25,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741884_1062 (size=5149) 2024-11-14T20:30:25,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741884_1062 (size=5149) 2024-11-14T20:30:25,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741884_1062 (size=5149) 2024-11-14T20:30:25,502 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/.tmp/a/3d9d5e6c938344b687da96f0b38afe26 2024-11-14T20:30:25,510 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/.tmp/a/3d9d5e6c938344b687da96f0b38afe26 as hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/a/3d9d5e6c938344b687da96f0b38afe26 2024-11-14T20:30:25,525 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/a/3d9d5e6c938344b687da96f0b38afe26, entries=1, sequenceid=5, filesize=5.0 K 2024-11-14T20:30:25,525 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for db92926d22002a42903150cc07bdb7f8 in 55ms, sequenceid=5, compaction requested=false; wal=null 2024-11-14T20:30:25,526 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/0000000000000000005 2024-11-14T20:30:25,527 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,527 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,528 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:25,530 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for db92926d22002a42903150cc07bdb7f8 2024-11-14T20:30:25,539 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/db92926d22002a42903150cc07bdb7f8/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-14T20:30:25,541 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened db92926d22002a42903150cc07bdb7f8; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65025389, jitterRate=-0.031046196818351746}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:25,541 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for db92926d22002a42903150cc07bdb7f8: Writing region info on filesystem at 1731616225442Initializing all the Stores at 1731616225445 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616225445Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616225445Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616225445Obtaining lock to block concurrent updates at 1731616225470 (+25 ms)Preparing flush snapshotting stores in db92926d22002a42903150cc07bdb7f8 at 1731616225470Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731616225470Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. at 1731616225471 (+1 ms)Flushing db92926d22002a42903150cc07bdb7f8/a: creating writer at 1731616225471Flushing db92926d22002a42903150cc07bdb7f8/a: appending metadata at 1731616225490 (+19 ms)Flushing db92926d22002a42903150cc07bdb7f8/a: closing flushed file at 1731616225490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2750e364: reopening flushed file at 1731616225509 (+19 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for db92926d22002a42903150cc07bdb7f8 in 55ms, sequenceid=5, compaction requested=false; wal=null at 1731616225525 (+16 ms)Cleaning up temporary data from old regions at 1731616225527 (+2 ms)Region opened successfully at 1731616225541 (+14 ms) 2024-11-14T20:30:25,546 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing db92926d22002a42903150cc07bdb7f8, disabling compactions & flushes 2024-11-14T20:30:25,546 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:25,546 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:25,546 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. after waiting 0 ms 2024-11-14T20:30:25,546 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:25,555 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731616220965.db92926d22002a42903150cc07bdb7f8. 2024-11-14T20:30:25,555 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for db92926d22002a42903150cc07bdb7f8: Waiting for close lock at 1731616225546Disabling compacts and flushes for region at 1731616225546Disabling writes for close at 1731616225546Writing region close event to WAL at 1731616225555 (+9 ms)Closed at 1731616225555 2024-11-14T20:30:25,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741883_1061 (size=95) 2024-11-14T20:30:25,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741883_1061 (size=95) 2024-11-14T20:30:25,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741883_1061 (size=95) 2024-11-14T20:30:25,591 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:25,591 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731616225411) 2024-11-14T20:30:25,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=409 (was 403) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1459261309_22 at /127.0.0.1:36178 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43677 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1459261309_22 at /127.0.0.1:41128 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43677 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1022 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=683 (was 690), ProcessCount=11 (was 11), AvailableMemoryMB=4111 (was 4360) 2024-11-14T20:30:25,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=409, OpenFileDescriptor=1022, MaxFileDescriptor=1048576, SystemLoadAverage=683, ProcessCount=11, AvailableMemoryMB=4109 2024-11-14T20:30:25,656 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:25,662 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-14T20:30:25,668 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6c8768110af2,36945,1731616203247 2024-11-14T20:30:25,671 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@207e022c 2024-11-14T20:30:25,673 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-14T20:30:25,676 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-14T20:30:25,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-14T20:30:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T20:30:25,705 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-14T20:30:25,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-14T20:30:25,708 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:25,711 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-14T20:30:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T20:30:25,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741885_1063 (size=694) 2024-11-14T20:30:25,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741885_1063 (size=694) 2024-11-14T20:30:25,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741885_1063 (size=694) 2024-11-14T20:30:25,769 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 18f7838e8e8a18bc6e359dc30256cd47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668 2024-11-14T20:30:25,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741886_1064 (size=77) 2024-11-14T20:30:25,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741886_1064 (size=77) 2024-11-14T20:30:25,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741886_1064 (size=77) 2024-11-14T20:30:25,804 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:25,804 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 18f7838e8e8a18bc6e359dc30256cd47, disabling compactions & flushes 2024-11-14T20:30:25,804 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:25,804 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:25,804 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. after waiting 0 ms 2024-11-14T20:30:25,804 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:25,804 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:25,804 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 18f7838e8e8a18bc6e359dc30256cd47: Waiting for close lock at 1731616225804Disabling compacts and flushes for region at 1731616225804Disabling writes for close at 1731616225804Writing region close event to WAL at 1731616225804Closed at 1731616225804 2024-11-14T20:30:25,807 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-14T20:30:25,813 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731616225807"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731616225807"}]},"ts":"1731616225807"} 2024-11-14T20:30:25,820 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-14T20:30:25,824 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-14T20:30:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T20:30:25,834 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731616225824"}]},"ts":"1731616225824"} 2024-11-14T20:30:25,844 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-14T20:30:25,845 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {6c8768110af2=0} racks are {/default-rack=0} 2024-11-14T20:30:25,849 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T20:30:25,849 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T20:30:25,849 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-14T20:30:25,849 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T20:30:25,849 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T20:30:25,849 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-14T20:30:25,849 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T20:30:25,849 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T20:30:25,849 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-14T20:30:25,849 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T20:30:25,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN}] 2024-11-14T20:30:25,856 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN 2024-11-14T20:30:25,859 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN; state=OFFLINE, location=6c8768110af2,42163,1731616204417; forceNewPlan=false, retain=false 2024-11-14T20:30:26,012 INFO [6c8768110af2:36945 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T20:30:26,012 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPENING, regionLocation=6c8768110af2,42163,1731616204417 2024-11-14T20:30:26,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN because future has completed 2024-11-14T20:30:26,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417}] 2024-11-14T20:30:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T20:30:26,178 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,178 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 18f7838e8e8a18bc6e359dc30256cd47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:26,179 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,179 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:26,179 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,179 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,183 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,185 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf1 2024-11-14T20:30:26,185 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:26,185 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:26,186 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,188 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf2 2024-11-14T20:30:26,188 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:26,189 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:26,189 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,190 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,190 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,192 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,192 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,193 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T20:30:26,195 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,198 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:26,199 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 18f7838e8e8a18bc6e359dc30256cd47; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71584638, jitterRate=0.0666942298412323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T20:30:26,199 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,200 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 18f7838e8e8a18bc6e359dc30256cd47: Running coprocessor pre-open hook at 1731616226179Writing region info on filesystem at 1731616226180 (+1 ms)Initializing all the Stores at 1731616226182 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616226182Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616226183 (+1 ms)Cleaning up temporary data from old regions at 1731616226192 (+9 ms)Running coprocessor post-open hooks at 1731616226199 (+7 ms)Region opened successfully at 1731616226200 (+1 ms) 2024-11-14T20:30:26,204 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., pid=6, masterSystemTime=1731616226171 2024-11-14T20:30:26,207 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,208 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,209 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPEN, openSeqNum=2, regionLocation=6c8768110af2,42163,1731616204417 2024-11-14T20:30:26,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417 because future has completed 2024-11-14T20:30:26,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-14T20:30:26,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417 in 197 msec 2024-11-14T20:30:26,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-14T20:30:26,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN in 367 msec 2024-11-14T20:30:26,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-14T20:30:26,227 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731616226227"}]},"ts":"1731616226227"} 2024-11-14T20:30:26,230 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-14T20:30:26,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-14T20:30:26,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 549 msec 2024-11-14T20:30:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-14T20:30:26,351 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-14T20:30:26,351 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-14T20:30:26,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T20:30:26,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-14T20:30:26,360 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-14T20:30:26,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-14T20:30:26,381 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=2] 2024-11-14T20:30:26,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=18f7838e8e8a18bc6e359dc30256cd47, source=6c8768110af2,42163,1731616204417, destination=6c8768110af2,46509,1731616204216, warming up region on 6c8768110af2,46509,1731616204216 2024-11-14T20:30:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T20:30:26,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=18f7838e8e8a18bc6e359dc30256cd47, source=6c8768110af2,42163,1731616204417, destination=6c8768110af2,46509,1731616204216, running balancer 2024-11-14T20:30:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE 2024-11-14T20:30:26,414 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE 2024-11-14T20:30:26,417 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47123, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T20:30:26,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=CLOSING, regionLocation=6c8768110af2,42163,1731616204417 2024-11-14T20:30:26,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE because future has completed 2024-11-14T20:30:26,445 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T20:30:26,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417}] 2024-11-14T20:30:26,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(7855): Warmup {ENCODED => 18f7838e8e8a18bc6e359dc30256cd47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:26,459 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,461 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf1 2024-11-14T20:30:26,461 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:26,461 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:26,462 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,463 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf2 2024-11-14T20:30:26,463 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:26,464 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(1722): Closing 18f7838e8e8a18bc6e359dc30256cd47, disabling compactions & flushes 2024-11-14T20:30:26,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. after waiting 0 ms 2024-11-14T20:30:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-14T20:30:26,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] regionserver.HRegion(1676): Region close journal for 18f7838e8e8a18bc6e359dc30256cd47: Waiting for close lock at 1731616226464Disabling compacts and flushes for region at 1731616226464Disabling writes for close at 1731616226464Writing region close event to WAL at 1731616226479 (+15 ms)Closed at 1731616226479 2024-11-14T20:30:26,629 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:26,629 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T20:30:26,630 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 18f7838e8e8a18bc6e359dc30256cd47, disabling compactions & flushes 2024-11-14T20:30:26,630 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,630 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,630 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. after waiting 0 ms 2024-11-14T20:30:26,630 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:26,630 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 18f7838e8e8a18bc6e359dc30256cd47 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-14T20:30:26,677 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf1/6000e7b8f7434e9b948a565047bc903a is 35, key is r1/cf1:q/1731616226385/Put/seqid=0 2024-11-14T20:30:26,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741887_1065 (size=4783) 2024-11-14T20:30:26,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741887_1065 (size=4783) 2024-11-14T20:30:26,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741887_1065 (size=4783) 2024-11-14T20:30:27,109 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf1/6000e7b8f7434e9b948a565047bc903a 2024-11-14T20:30:27,119 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf1/6000e7b8f7434e9b948a565047bc903a as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a 2024-11-14T20:30:27,132 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a, entries=1, sequenceid=5, filesize=4.7 K 2024-11-14T20:30:27,134 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 18f7838e8e8a18bc6e359dc30256cd47 in 504ms, sequenceid=5, compaction requested=false 2024-11-14T20:30:27,134 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-14T20:30:27,146 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T20:30:27,149 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:27,149 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 18f7838e8e8a18bc6e359dc30256cd47: Waiting for close lock at 1731616226630Running coprocessor pre-close hooks at 1731616226630Disabling compacts and flushes for region at 1731616226630Disabling writes for close at 1731616226630Obtaining lock to block concurrent updates at 1731616226630Preparing flush snapshotting stores in 18f7838e8e8a18bc6e359dc30256cd47 at 1731616226630Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731616226631 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. at 1731616226632 (+1 ms)Flushing 18f7838e8e8a18bc6e359dc30256cd47/cf1: creating writer at 1731616226632Flushing 18f7838e8e8a18bc6e359dc30256cd47/cf1: appending metadata at 1731616226677 (+45 ms)Flushing 18f7838e8e8a18bc6e359dc30256cd47/cf1: closing flushed file at 1731616226677Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@732c5ce2: reopening flushed file at 1731616227118 (+441 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 18f7838e8e8a18bc6e359dc30256cd47 in 504ms, sequenceid=5, compaction requested=false at 1731616227134 (+16 ms)Writing region close event to WAL at 1731616227141 (+7 ms)Running coprocessor post-close hooks at 1731616227147 (+6 ms)Closed at 1731616227149 (+2 ms) 2024-11-14T20:30:27,150 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 18f7838e8e8a18bc6e359dc30256cd47 move to 6c8768110af2,46509,1731616204216 record at close sequenceid=5 2024-11-14T20:30:27,154 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,155 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=CLOSED 2024-11-14T20:30:27,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417 because future has completed 2024-11-14T20:30:27,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-14T20:30:27,164 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417 in 715 msec 2024-11-14T20:30:27,166 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE; state=CLOSED, location=6c8768110af2,46509,1731616204216; forceNewPlan=false, retain=false 2024-11-14T20:30:27,319 INFO [6c8768110af2:36945 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T20:30:27,319 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPENING, regionLocation=6c8768110af2,46509,1731616204216 2024-11-14T20:30:27,322 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE because future has completed 2024-11-14T20:30:27,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216}] 2024-11-14T20:30:27,487 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:27,487 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 18f7838e8e8a18bc6e359dc30256cd47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:27,488 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,488 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:27,488 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,488 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,490 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,491 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf1 2024-11-14T20:30:27,492 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:27,518 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a 2024-11-14T20:30:27,518 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:27,519 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,527 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf2 2024-11-14T20:30:27,527 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:27,528 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:27,528 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,533 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,534 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,535 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,535 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,535 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T20:30:27,539 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,540 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 18f7838e8e8a18bc6e359dc30256cd47; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67276642, jitterRate=0.002500087022781372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T20:30:27,540 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,540 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 18f7838e8e8a18bc6e359dc30256cd47: Running coprocessor pre-open hook at 1731616227488Writing region info on filesystem at 1731616227488Initializing all the Stores at 1731616227490 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616227490Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616227490Cleaning up temporary data from old regions at 1731616227535 (+45 ms)Running coprocessor post-open hooks at 1731616227540 (+5 ms)Region opened successfully at 1731616227540 2024-11-14T20:30:27,542 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., pid=9, masterSystemTime=1731616227477 2024-11-14T20:30:27,546 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:27,546 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:27,549 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPEN, openSeqNum=9, regionLocation=6c8768110af2,46509,1731616204216 2024-11-14T20:30:27,558 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216 because future has completed 2024-11-14T20:30:27,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-14T20:30:27,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216 in 241 msec 2024-11-14T20:30:27,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE in 1.1570 sec 2024-11-14T20:30:27,623 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T20:30:27,627 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T20:30:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:54466 deadline: 1731616287632, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=46509 startCode=1731616204216. As of locationSeqNum=5. 2024-11-14T20:30:27,641 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=46509 startCode=1731616204216. As of locationSeqNum=5. 2024-11-14T20:30:27,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=46509 startCode=1731616204216. As of locationSeqNum=5. 2024-11-14T20:30:27,642 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,46509,1731616204216, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=46509 startCode=1731616204216. As of locationSeqNum=5. 2024-11-14T20:30:27,750 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T20:30:27,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T20:30:27,766 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 18f7838e8e8a18bc6e359dc30256cd47 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-14T20:30:27,784 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf1/b22f7f7fc86844cbb34d565efb32c078 is 29, key is r1/cf1:/1731616227754/DeleteFamily/seqid=0 2024-11-14T20:30:27,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741888_1066 (size=4906) 2024-11-14T20:30:27,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741888_1066 (size=4906) 2024-11-14T20:30:27,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741888_1066 (size=4906) 2024-11-14T20:30:27,802 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf1/b22f7f7fc86844cbb34d565efb32c078 2024-11-14T20:30:27,809 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b22f7f7fc86844cbb34d565efb32c078 2024-11-14T20:30:27,831 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf2/9ee8a59c999d47049b89580d8fff36f0 is 29, key is r1/cf2:/1731616227754/DeleteFamily/seqid=0 2024-11-14T20:30:27,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741889_1067 (size=4906) 2024-11-14T20:30:27,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741889_1067 (size=4906) 2024-11-14T20:30:27,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741889_1067 (size=4906) 2024-11-14T20:30:27,841 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf2/9ee8a59c999d47049b89580d8fff36f0 2024-11-14T20:30:27,848 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9ee8a59c999d47049b89580d8fff36f0 2024-11-14T20:30:27,849 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf1/b22f7f7fc86844cbb34d565efb32c078 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b22f7f7fc86844cbb34d565efb32c078 2024-11-14T20:30:27,856 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b22f7f7fc86844cbb34d565efb32c078 2024-11-14T20:30:27,856 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b22f7f7fc86844cbb34d565efb32c078, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T20:30:27,857 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf2/9ee8a59c999d47049b89580d8fff36f0 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/9ee8a59c999d47049b89580d8fff36f0 2024-11-14T20:30:27,864 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9ee8a59c999d47049b89580d8fff36f0 2024-11-14T20:30:27,864 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/9ee8a59c999d47049b89580d8fff36f0, entries=1, sequenceid=12, filesize=4.8 K 2024-11-14T20:30:27,866 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 18f7838e8e8a18bc6e359dc30256cd47 in 100ms, sequenceid=12, compaction requested=false 2024-11-14T20:30:27,866 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 18f7838e8e8a18bc6e359dc30256cd47: 2024-11-14T20:30:27,869 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-14T20:30:27,871 DEBUG [Time-limited test {}] regionserver.HStore(1541): 18f7838e8e8a18bc6e359dc30256cd47/cf1 is initiating major compaction (all files) 2024-11-14T20:30:27,871 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T20:30:27,871 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:27,871 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 18f7838e8e8a18bc6e359dc30256cd47/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:27,872 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a, hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b22f7f7fc86844cbb34d565efb32c078] into tmpdir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp, totalSize=9.5 K 2024-11-14T20:30:27,873 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6000e7b8f7434e9b948a565047bc903a, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731616226385 2024-11-14T20:30:27,874 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b22f7f7fc86844cbb34d565efb32c078, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T20:30:27,887 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 18f7838e8e8a18bc6e359dc30256cd47#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T20:30:27,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741890_1068 (size=4626) 2024-11-14T20:30:27,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741890_1068 (size=4626) 2024-11-14T20:30:27,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741890_1068 (size=4626) 2024-11-14T20:30:27,903 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf1/b3026c2cbb3a42c7870a1fec4bec7f98 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b3026c2cbb3a42c7870a1fec4bec7f98 2024-11-14T20:30:27,922 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 18f7838e8e8a18bc6e359dc30256cd47/cf1 of 18f7838e8e8a18bc6e359dc30256cd47 into b3026c2cbb3a42c7870a1fec4bec7f98(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T20:30:27,922 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 18f7838e8e8a18bc6e359dc30256cd47: 2024-11-14T20:30:27,922 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-14T20:30:27,922 DEBUG [Time-limited test {}] regionserver.HStore(1541): 18f7838e8e8a18bc6e359dc30256cd47/cf2 is initiating major compaction (all files) 2024-11-14T20:30:27,922 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-14T20:30:27,922 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-14T20:30:27,922 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 18f7838e8e8a18bc6e359dc30256cd47/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:27,923 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/9ee8a59c999d47049b89580d8fff36f0] into tmpdir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp, totalSize=4.8 K 2024-11-14T20:30:27,923 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9ee8a59c999d47049b89580d8fff36f0, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-14T20:30:27,932 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 18f7838e8e8a18bc6e359dc30256cd47#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-14T20:30:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741891_1069 (size=4592) 2024-11-14T20:30:27,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741891_1069 (size=4592) 2024-11-14T20:30:27,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741891_1069 (size=4592) 2024-11-14T20:30:27,962 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/.tmp/cf2/bc12c42e80b146359be4bf6eeffdb370 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/bc12c42e80b146359be4bf6eeffdb370 2024-11-14T20:30:27,977 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 18f7838e8e8a18bc6e359dc30256cd47/cf2 of 18f7838e8e8a18bc6e359dc30256cd47 into bc12c42e80b146359be4bf6eeffdb370(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T20:30:27,977 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 18f7838e8e8a18bc6e359dc30256cd47: 2024-11-14T20:30:27,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=18f7838e8e8a18bc6e359dc30256cd47, source=6c8768110af2,46509,1731616204216, destination=6c8768110af2,42163,1731616204417, warming up region on 6c8768110af2,42163,1731616204417 2024-11-14T20:30:27,983 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=18f7838e8e8a18bc6e359dc30256cd47, source=6c8768110af2,46509,1731616204216, destination=6c8768110af2,42163,1731616204417, running balancer 2024-11-14T20:30:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE 2024-11-14T20:30:27,985 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE 2024-11-14T20:30:27,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(7855): Warmup {ENCODED => 18f7838e8e8a18bc6e359dc30256cd47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:27,987 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=CLOSING, regionLocation=6c8768110af2,46509,1731616204216 2024-11-14T20:30:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:27,988 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:27,990 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf1 2024-11-14T20:30:27,990 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:27,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE because future has completed 2024-11-14T20:30:27,991 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-14T20:30:27,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216}] 2024-11-14T20:30:28,008 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a 2024-11-14T20:30:28,013 INFO [StoreFileOpener-18f7838e8e8a18bc6e359dc30256cd47-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b22f7f7fc86844cbb34d565efb32c078 2024-11-14T20:30:28,013 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b22f7f7fc86844cbb34d565efb32c078 2024-11-14T20:30:28,022 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b3026c2cbb3a42c7870a1fec4bec7f98 2024-11-14T20:30:28,023 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:28,023 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,025 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf2 2024-11-14T20:30:28,026 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:28,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-14T20:30:28,037 INFO [StoreFileOpener-18f7838e8e8a18bc6e359dc30256cd47-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9ee8a59c999d47049b89580d8fff36f0 2024-11-14T20:30:28,038 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/9ee8a59c999d47049b89580d8fff36f0 2024-11-14T20:30:28,044 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/bc12c42e80b146359be4bf6eeffdb370 2024-11-14T20:30:28,045 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(1722): Closing 18f7838e8e8a18bc6e359dc30256cd47, disabling compactions & flushes 2024-11-14T20:30:28,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. after waiting 0 ms 2024-11-14T20:30:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42163 {}] regionserver.HRegion(1676): Region close journal for 18f7838e8e8a18bc6e359dc30256cd47: Waiting for close lock at 1731616228045Disabling compacts and flushes for region at 1731616228045Disabling writes for close at 1731616228045Writing region close event to WAL at 1731616228046 (+1 ms)Closed at 1731616228046 2024-11-14T20:30:28,145 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,145 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-14T20:30:28,146 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 18f7838e8e8a18bc6e359dc30256cd47, disabling compactions & flushes 2024-11-14T20:30:28,146 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,146 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,146 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. after waiting 0 ms 2024-11-14T20:30:28,146 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,146 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a, hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b22f7f7fc86844cbb34d565efb32c078] to archive 2024-11-14T20:30:28,150 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T20:30:28,154 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/6000e7b8f7434e9b948a565047bc903a 2024-11-14T20:30:28,156 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b22f7f7fc86844cbb34d565efb32c078 to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b22f7f7fc86844cbb34d565efb32c078 2024-11-14T20:30:28,172 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/9ee8a59c999d47049b89580d8fff36f0] to archive 2024-11-14T20:30:28,184 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T20:30:28,193 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/9ee8a59c999d47049b89580d8fff36f0 to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/9ee8a59c999d47049b89580d8fff36f0 2024-11-14T20:30:28,204 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-14T20:30:28,208 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,208 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 18f7838e8e8a18bc6e359dc30256cd47: Waiting for close lock at 1731616228145Running coprocessor pre-close hooks at 1731616228145Disabling compacts and flushes for region at 1731616228145Disabling writes for close at 1731616228146 (+1 ms)Writing region close event to WAL at 1731616228197 (+51 ms)Running coprocessor post-close hooks at 1731616228208 (+11 ms)Closed at 1731616228208 2024-11-14T20:30:28,208 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 18f7838e8e8a18bc6e359dc30256cd47 move to 6c8768110af2,42163,1731616204417 record at close sequenceid=12 2024-11-14T20:30:28,213 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,217 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=CLOSED 2024-11-14T20:30:28,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216 because future has completed 2024-11-14T20:30:28,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-14T20:30:28,225 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216 in 231 msec 2024-11-14T20:30:28,226 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE; state=CLOSED, location=6c8768110af2,42163,1731616204417; forceNewPlan=false, retain=false 2024-11-14T20:30:28,377 INFO [6c8768110af2:36945 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-14T20:30:28,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPENING, regionLocation=6c8768110af2,42163,1731616204417 2024-11-14T20:30:28,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE because future has completed 2024-11-14T20:30:28,381 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417}] 2024-11-14T20:30:28,538 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,538 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 18f7838e8e8a18bc6e359dc30256cd47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:28,539 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,539 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:28,539 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,539 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,540 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,541 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf1 2024-11-14T20:30:28,541 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:28,548 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b3026c2cbb3a42c7870a1fec4bec7f98 2024-11-14T20:30:28,549 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:28,549 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,549 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf2 2024-11-14T20:30:28,550 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:28,559 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/bc12c42e80b146359be4bf6eeffdb370 2024-11-14T20:30:28,559 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:28,559 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,560 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,561 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,562 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,562 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,562 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T20:30:28,564 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,565 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 18f7838e8e8a18bc6e359dc30256cd47; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71733879, jitterRate=0.06891809403896332}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T20:30:28,565 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,565 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 18f7838e8e8a18bc6e359dc30256cd47: Running coprocessor pre-open hook at 1731616228539Writing region info on filesystem at 1731616228539Initializing all the Stores at 1731616228540 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616228540Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616228540Cleaning up temporary data from old regions at 1731616228562 (+22 ms)Running coprocessor post-open hooks at 1731616228565 (+3 ms)Region opened successfully at 1731616228565 2024-11-14T20:30:28,566 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., pid=12, masterSystemTime=1731616228534 2024-11-14T20:30:28,569 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,570 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,571 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPEN, openSeqNum=18, regionLocation=6c8768110af2,42163,1731616204417 2024-11-14T20:30:28,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417 because future has completed 2024-11-14T20:30:28,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-14T20:30:28,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,42163,1731616204417 in 197 msec 2024-11-14T20:30:28,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, REOPEN/MOVE in 598 msec 2024-11-14T20:30:28,586 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T20:30:28,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T20:30:28,590 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 6c8768110af2,42163,1731616204417: testing ***** 2024-11-14T20:30:28,590 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-14T20:30:28,592 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-14T20:30:28,593 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-14T20:30:28,599 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-14T20:30:28,602 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-14T20:30:28,628 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 288035944 }, "NonHeapMemoryUsage": { "committed": 172359680, "init": 7667712, "max": -1, "used": 169751656 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "6c8768110af2", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 64, "ProcessCallTime_mean": 7, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 60, "ProcessCallTime_98th_percentile": 64, "ProcessCallTime_99th_percentile": 64, "ProcessCallTime_99.9th_percentile": 64, "ProcessCallTime_TimeRangeCount_0-1": 7, "ProcessCallTime_TimeRangeCount_1-3": 11, "ProcessCallTime_TimeRangeCount_3-10": 10, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 2, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 64, "TotalCallTime_mean": 7, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 6, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 61, "TotalCallTime_98th_percentile": 64, "TotalCallTime_99th_percentile": 64, "TotalCallTime_99.9th_percentile": 64, "TotalCallTime_TimeRangeCount_0-1": 7, "TotalCallTime_TimeRangeCount_1-3": 11, "TotalCallTime_TimeRangeCount_3-10": 10, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 2, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5782 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "6c8768110af2", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:60776", "tag.serverName": "6c8768110af2,46509,1731616204216", "tag.clusterId": "0a234c43-7201-4428-9c4c-8a6f53c029dd", "tag.Context": "regionserver", "tag.Hostname": "6c8768110af2", "regionCount": 0, "storeCount": 0, "hlogFileCount": 4, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731616204216, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 3, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 1, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 47, "MajorCompactionTime_max": 55, "MajorCompactionTime_mean": 51, "MajorCompactionTime_25th_percentile": 47, "MajorCompactionTime_median": 48, "MajorCompactionTime_75th_percentile": 55, "MajorCompactionTime_90th_percentile": 55, "MajorCompactionTime_95th_percentile": 55, "MajorCompactionTime_98th_percentile": 55, "MajorCompactionTime_99th_percentile": 55, "MajorCompactionTime_99.9th_percentile": 55, "MajorCompactionTime_TimeRangeCount_30-100": 2, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 4, "Delete_max": 4, "Delete_mean": 4, "Delete_25th_percentile": 4, "Delete_median": 4, "Delete_75th_percentile": 4, "Delete_90th_percentile": 4, "Delete_95th_percentile": 4, "Delete_98th_percentile": 4, "Delete_99th_percentile": 4, "Delete_99.9th_percentile": 4, "Delete_TimeRangeCount_3-10": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 47, "CompactionTime_max": 55, "CompactionTime_mean": 51, "CompactionTime_25th_percentile": 47, "CompactionTime_median": 48, "CompactionTime_75th_percentile": 55, "CompactionTime_90th_percentile": 55, "CompactionTime_95th_percentile": 55, "CompactionTime_98th_percentile": 55, "CompactionTime_99th_percentile": 55, "CompactionTime_99.9th_percentile": 55, "CompactionTime_TimeRangeCount_30-100": 2, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 100, "FlushTime_max": 100, "FlushTime_mean": 100, "FlushTime_25th_percentile": 100, "FlushTime_median": 100, "FlushTime_75th_percentile": 100, "FlushTime_90th_percentile": 100, "FlushTime_95th_percentile": 100, "FlushTime_98th_percentile": 100, "FlushTime_99th_percentile": 100, "FlushTime_99.9th_percentile": 100, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 1, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-14T20:30:28,653 WARN [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36945 {}] master.MasterRpcServices(700): 6c8768110af2,42163,1731616204417 reported a fatal error: ***** ABORTING region server 6c8768110af2,42163,1731616204417: testing ***** 2024-11-14T20:30:28,673 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6c8768110af2,42163,1731616204417' ***** 2024-11-14T20:30:28,673 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-14T20:30:28,673 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T20:30:28,674 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T20:30:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46509 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:43046 deadline: 1731616288674, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=42163 startCode=1731616204417. As of locationSeqNum=12. 2024-11-14T20:30:28,675 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,46509,1731616204216, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,46509,1731616204216, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=42163 startCode=1731616204417. As of locationSeqNum=12. 2024-11-14T20:30:28,676 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,46509,1731616204216, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=42163 startCode=1731616204417. As of locationSeqNum=12. 2024-11-14T20:30:28,676 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,46509,1731616204216, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=6c8768110af2 port=42163 startCode=1731616204417. As of locationSeqNum=12. 2024-11-14T20:30:28,676 INFO [RS:2;6c8768110af2:42163 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-14T20:30:28,676 INFO [RS:2;6c8768110af2:42163 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-14T20:30:28,677 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(3091): Received CLOSE for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,677 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(956): aborting server 6c8768110af2,42163,1731616204417 2024-11-14T20:30:28,677 INFO [RS:2;6c8768110af2:42163 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T20:30:28,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6c8768110af2,42163,1731616204417 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:28,678 INFO [RS:2;6c8768110af2:42163 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;6c8768110af2:42163. 2024-11-14T20:30:28,678 DEBUG [RS:2;6c8768110af2:42163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T20:30:28,678 DEBUG [RS:2;6c8768110af2:42163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:28,679 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6c8768110af2,42163,1731616204417 aborting 2024-11-14T20:30:28,679 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6c8768110af2,42163,1731616204417 aborting 2024-11-14T20:30:28,679 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 from cache 2024-11-14T20:30:28,680 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 18f7838e8e8a18bc6e359dc30256cd47, disabling compactions & flushes 2024-11-14T20:30:28,680 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,680 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,680 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. after waiting 0 ms 2024-11-14T20:30:28,680 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,684 INFO [RS:2;6c8768110af2:42163 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T20:30:28,685 INFO [RS:2;6c8768110af2:42163 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T20:30:28,685 INFO [RS:2;6c8768110af2:42163 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T20:30:28,696 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T20:30:28,699 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T20:30:28,699 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T20:30:28,700 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 18f7838e8e8a18bc6e359dc30256cd47=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.} 2024-11-14T20:30:28,700 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T20:30:28,700 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T20:30:28,700 DEBUG [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:28,700 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T20:30:28,700 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T20:30:28,703 ERROR [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-14T20:30:28,704 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T20:30:28,704 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T20:30:28,704 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731616228699Running coprocessor pre-close hooks at 1731616228699Disabling compacts and flushes for region at 1731616228699Disabling writes for close at 1731616228700 (+1 ms)Writing region close event to WAL at 1731616228703 (+3 ms)Running coprocessor post-close hooks at 1731616228703Closed at 1731616228704 (+1 ms) 2024-11-14T20:30:28,704 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T20:30:28,708 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,709 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 18f7838e8e8a18bc6e359dc30256cd47: Waiting for close lock at 1731616228679Running coprocessor pre-close hooks at 1731616228680 (+1 ms)Disabling compacts and flushes for region at 1731616228680Disabling writes for close at 1731616228680Writing region close event to WAL at 1731616228708 (+28 ms)Running coprocessor post-close hooks at 1731616228708Closed at 1731616228708 2024-11-14T20:30:28,709 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:28,733 INFO [regionserver/6c8768110af2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T20:30:28,749 INFO [regionserver/6c8768110af2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-14T20:30:28,749 INFO [regionserver/6c8768110af2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-14T20:30:28,789 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:28,791 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1] 2024-11-14T20:30:28,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6c8768110af2,42163,1731616204417 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:28,794 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6c8768110af2,42163,1731616204417 aborting 2024-11-14T20:30:28,796 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 6c8768110af2,42163,1731616204417 aborting 2024-11-14T20:30:28,796 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 from cache 2024-11-14T20:30:28,901 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(976): stopping server 6c8768110af2,42163,1731616204417; all regions closed. 2024-11-14T20:30:28,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741836_1012 (size=3561) 2024-11-14T20:30:28,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741836_1012 (size=3561) 2024-11-14T20:30:28,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741836_1012 (size=3561) 2024-11-14T20:30:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741833_1009 (size=1397) 2024-11-14T20:30:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741833_1009 (size=1397) 2024-11-14T20:30:28,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741833_1009 (size=1397) 2024-11-14T20:30:28,915 DEBUG [RS:2;6c8768110af2:42163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:28,915 INFO [RS:2;6c8768110af2:42163 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T20:30:28,915 INFO [RS:2;6c8768110af2:42163 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T20:30:28,916 INFO [RS:2;6c8768110af2:42163 {}] hbase.ChoreService(370): Chore service for: regionserver/6c8768110af2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T20:30:28,916 INFO [RS:2;6c8768110af2:42163 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T20:30:28,916 INFO [regionserver/6c8768110af2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T20:30:28,916 INFO [RS:2;6c8768110af2:42163 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42163 2024-11-14T20:30:28,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6c8768110af2,42163,1731616204417 2024-11-14T20:30:28,962 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T20:30:28,962 INFO [RS:2;6c8768110af2:42163 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T20:30:28,999 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:29,001 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1] 2024-11-14T20:30:29,003 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6c8768110af2:42163 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6c8768110af2/172.17.0.3:42163 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:29,004 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1, error=java.net.ConnectException: Call to address=6c8768110af2:42163 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6c8768110af2/172.17.0.3:42163 2024-11-14T20:30:29,004 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-14T20:30:29,004 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 from cache 2024-11-14T20:30:29,005 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 6c8768110af2:42163 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 6c8768110af2/172.17.0.3:42163 2024-11-14T20:30:29,009 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6c8768110af2,42163,1731616204417] 2024-11-14T20:30:29,039 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6c8768110af2,42163,1731616204417 already deleted, retry=false 2024-11-14T20:30:29,039 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6c8768110af2,42163,1731616204417 on 6c8768110af2,36945,1731616203247 2024-11-14T20:30:29,047 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6c8768110af2,42163,1731616204417, splitWal=true, meta=true 2024-11-14T20:30:29,051 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6c8768110af2,42163,1731616204417, splitWal=true, meta=true 2024-11-14T20:30:29,052 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6c8768110af2,42163,1731616204417, splitWal=true, meta=true, isMeta: true 2024-11-14T20:30:29,055 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting 2024-11-14T20:30:29,056 INFO [PEWorker-5 {}] master.SplitWALManager(105): 6c8768110af2,42163,1731616204417 WAL count=1, meta=true 2024-11-14T20:30:29,060 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 6c8768110af2,42163,1731616204417 (carryingMeta=true) 6c8768110af2,42163,1731616204417/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@481fcacf[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-14T20:30:29,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta}] 2024-11-14T20:30:29,071 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=6c8768110af2,44657,1731616204351 2024-11-14T20:30:29,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, worker=6c8768110af2,44657,1731616204351}] 2024-11-14T20:30:29,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:29,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42163-0x1003ffc70d80003, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:29,110 INFO [RS:2;6c8768110af2:42163 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T20:30:29,110 INFO [RS:2;6c8768110af2:42163 {}] regionserver.HRegionServer(1031): Exiting; stopping=6c8768110af2,42163,1731616204417; zookeeper connection closed. 2024-11-14T20:30:29,111 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@14720e4a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@14720e4a 2024-11-14T20:30:29,236 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-14T20:30:29,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43965, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-14T20:30:29,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-14T20:30:29,267 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, size=3.5 K (3561bytes) 2024-11-14T20:30:29,267 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta 2024-11-14T20:30:29,268 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta after 1ms 2024-11-14T20:30:29,271 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:29,271 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta took 4ms 2024-11-14T20:30:29,298 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-14T20:30:29,299 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta so closing down 2024-11-14T20:30:29,299 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:29,301 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta.temp 2024-11-14T20:30:29,303 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta.temp 2024-11-14T20:30:29,304 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:29,309 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:29,310 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1] 2024-11-14T20:30:29,311 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6c8768110af2:42163 this server is in the failed servers list 2024-11-14T20:30:29,312 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6c8768110af2:42163 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6c8768110af2:42163 2024-11-14T20:30:29,312 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6c8768110af2:42163 2024-11-14T20:30:29,312 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 from cache 2024-11-14T20:30:29,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741892_1070 (size=3346) 2024-11-14T20:30:29,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741892_1070 (size=3346) 2024-11-14T20:30:29,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741892_1070 (size=3346) 2024-11-14T20:30:29,337 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:29,340 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta.temp to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T20:30:29,340 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 56 ms; skipped=1; WAL=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false 2024-11-14T20:30:29,341 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, journal: Splitting hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, size=3.5 K (3561bytes) at 1731616229267Finishing writing output for hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta so closing down at 1731616229299 (+32 ms)Creating recovered edits writer path=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta.temp at 1731616229303 (+4 ms)3 split writer threads finished at 1731616229304 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731616229338 (+34 ms)Rename recovered edits hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta.temp to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731616229340 (+2 ms)Processed 16 edits across 1 Regions in 56 ms; skipped=1; WAL=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false at 1731616229341 (+1 ms) 2024-11-14T20:30:29,341 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta 2024-11-14T20:30:29,343 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-14T20:30:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-14T20:30:29,355 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs 2024-11-14T20:30:29,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-14T20:30:29,360 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, worker=6c8768110af2,44657,1731616204351 in 282 msec 2024-11-14T20:30:29,363 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=6c8768110af2,44657,1731616204351 2024-11-14T20:30:29,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-14T20:30:29,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 6c8768110af2%2C42163%2C1731616204417.meta.1731616207379.meta, worker=6c8768110af2,44657,1731616204351 in 305 msec 2024-11-14T20:30:29,374 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting dir is empty, no logs to split. 2024-11-14T20:30:29,374 INFO [PEWorker-5 {}] master.SplitWALManager(105): 6c8768110af2,42163,1731616204417 WAL count=0, meta=true 2024-11-14T20:30:29,374 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 6c8768110af2,42163,1731616204417 WAL splitting is done? wals=0, meta=true 2024-11-14T20:30:29,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-14T20:30:29,379 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T20:30:29,380 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T20:30:29,530 DEBUG [6c8768110af2:36945 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-14T20:30:29,531 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(204): Hosts are {6c8768110af2=0} racks are {/default-rack=0} 2024-11-14T20:30:29,531 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T20:30:29,531 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T20:30:29,531 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T20:30:29,531 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T20:30:29,531 INFO [6c8768110af2:36945 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T20:30:29,531 INFO [6c8768110af2:36945 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T20:30:29,531 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T20:30:29,532 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6c8768110af2,46509,1731616204216 2024-11-14T20:30:29,534 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6c8768110af2,46509,1731616204216, state=OPENING 2024-11-14T20:30:29,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:29,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:29,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:29,537 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:29,537 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-14T20:30:29,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6c8768110af2,46509,1731616204216}] 2024-11-14T20:30:29,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:29,538 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:29,697 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-14T20:30:29,697 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:29,698 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-14T20:30:29,700 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6c8768110af2%2C46509%2C1731616204216.meta, suffix=.meta, logDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216, archiveDir=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs, maxLogs=32 2024-11-14T20:30:29,713 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216/6c8768110af2%2C46509%2C1731616204216.meta.1731616229700.meta, exclude list is [], retry=0 2024-11-14T20:30:29,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:29,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:29,717 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:29,719 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216/6c8768110af2%2C46509%2C1731616204216.meta.1731616229700.meta 2024-11-14T20:30:29,720 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:29,720 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:29,721 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-14T20:30:29,721 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-14T20:30:29,721 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-14T20:30:29,721 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-14T20:30:29,721 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:29,721 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-14T20:30:29,722 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-14T20:30:29,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-14T20:30:29,726 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-14T20:30:29,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:29,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:29,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-14T20:30:29,728 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-14T20:30:29,728 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:29,728 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:29,729 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-14T20:30:29,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-14T20:30:29,730 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:29,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:29,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-14T20:30:29,731 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-14T20:30:29,731 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:29,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-14T20:30:29,732 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-14T20:30:29,733 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740 2024-11-14T20:30:29,735 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740 2024-11-14T20:30:29,735 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T20:30:29,738 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:29,742 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T20:30:29,742 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-14T20:30:29,763 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/info/e016f06b6b794c388985d00766665642 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47./info:regioninfo/1731616228570/Put/seqid=0 2024-11-14T20:30:29,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741894_1072 (size=11177) 2024-11-14T20:30:29,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741894_1072 (size=11177) 2024-11-14T20:30:29,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741894_1072 (size=11177) 2024-11-14T20:30:29,776 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/info/e016f06b6b794c388985d00766665642 2024-11-14T20:30:29,809 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/ns/ef320a476dd14c469c00da94e3a69860 is 43, key is default/ns:d/1731616207619/Put/seqid=0 2024-11-14T20:30:29,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741895_1073 (size=5153) 2024-11-14T20:30:29,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741895_1073 (size=5153) 2024-11-14T20:30:29,819 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:29,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741895_1073 (size=5153) 2024-11-14T20:30:29,820 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/ns/ef320a476dd14c469c00da94e3a69860 2024-11-14T20:30:29,820 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,46509,1731616204216, seqNum=-1] 2024-11-14T20:30:29,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46509 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.3:43046 deadline: 1731616289821, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6c8768110af2,46509,1731616204216 2024-11-14T20:30:29,822 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,46509,1731616204216, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6c8768110af2,46509,1731616204216, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6c8768110af2,46509,1731616204216 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T20:30:29,822 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,46509,1731616204216, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on 6c8768110af2,46509,1731616204216 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-14T20:30:29,822 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6c8768110af2,46509,1731616204216, seqNum=-1 from cache 2024-11-14T20:30:29,849 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/table/ffb71ff2589643b793bc9d0709d8f6fc is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731616226227/Put/seqid=0 2024-11-14T20:30:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741896_1074 (size=5431) 2024-11-14T20:30:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741896_1074 (size=5431) 2024-11-14T20:30:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741896_1074 (size=5431) 2024-11-14T20:30:29,863 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/table/ffb71ff2589643b793bc9d0709d8f6fc 2024-11-14T20:30:29,873 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/info/e016f06b6b794c388985d00766665642 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/info/e016f06b6b794c388985d00766665642 2024-11-14T20:30:29,886 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/info/e016f06b6b794c388985d00766665642, entries=36, sequenceid=18, filesize=10.9 K 2024-11-14T20:30:29,887 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/ns/ef320a476dd14c469c00da94e3a69860 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/ns/ef320a476dd14c469c00da94e3a69860 2024-11-14T20:30:29,895 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/ns/ef320a476dd14c469c00da94e3a69860, entries=2, sequenceid=18, filesize=5.0 K 2024-11-14T20:30:29,901 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/table/ffb71ff2589643b793bc9d0709d8f6fc as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/table/ffb71ff2589643b793bc9d0709d8f6fc 2024-11-14T20:30:29,923 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/table/ffb71ff2589643b793bc9d0709d8f6fc, entries=2, sequenceid=18, filesize=5.3 K 2024-11-14T20:30:29,923 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 181ms, sequenceid=18, compaction requested=false; wal=null 2024-11-14T20:30:29,927 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-14T20:30:29,931 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-14T20:30:29,931 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-14T20:30:29,935 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-14T20:30:29,938 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-14T20:30:29,948 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-14T20:30:29,950 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64791066, jitterRate=-0.03453788161277771}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-14T20:30:29,950 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-14T20:30:29,951 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731616229722Writing region info on filesystem at 1731616229722Initializing all the Stores at 1731616229723 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616229723Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616229724 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616229724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731616229724Obtaining lock to block concurrent updates at 1731616229742 (+18 ms)Preparing flush snapshotting stores in 1588230740 at 1731616229742Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731616229742Flushing stores of hbase:meta,,1.1588230740 at 1731616229742Flushing 1588230740/info: creating writer at 1731616229743 (+1 ms)Flushing 1588230740/info: appending metadata at 1731616229762 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731616229763 (+1 ms)Flushing 1588230740/ns: creating writer at 1731616229784 (+21 ms)Flushing 1588230740/ns: appending metadata at 1731616229808 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1731616229808Flushing 1588230740/table: creating writer at 1731616229827 (+19 ms)Flushing 1588230740/table: appending metadata at 1731616229849 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731616229849Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e6b962a: reopening flushed file at 1731616229870 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34f918f4: reopening flushed file at 1731616229886 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ee69193: reopening flushed file at 1731616229895 (+9 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 181ms, sequenceid=18, compaction requested=false; wal=null at 1731616229923 (+28 ms)Cleaning up temporary data from old regions at 1731616229931 (+8 ms)Running coprocessor post-open hooks at 1731616229950 (+19 ms)Region opened successfully at 1731616229951 (+1 ms) 2024-11-14T20:30:29,959 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731616229692 2024-11-14T20:30:29,968 DEBUG [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-14T20:30:29,968 INFO [RS_OPEN_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-14T20:30:29,968 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=6c8768110af2,46509,1731616204216 2024-11-14T20:30:29,970 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6c8768110af2,46509,1731616204216, state=OPEN 2024-11-14T20:30:29,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:29,977 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:29,977 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6c8768110af2,46509,1731616204216 2024-11-14T20:30:29,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:29,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-14T20:30:29,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:29,981 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-14T20:30:29,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-14T20:30:29,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6c8768110af2,46509,1731616204216 in 440 msec 2024-11-14T20:30:29,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-14T20:30:29,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-11-14T20:30:29,990 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): 6c8768110af2,42163,1731616204417 had 2 regions 2024-11-14T20:30:29,992 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6c8768110af2,42163,1731616204417, splitWal=true, meta=true, isMeta: false 2024-11-14T20:30:29,999 INFO [PEWorker-5 {}] master.SplitWALManager(105): 6c8768110af2,42163,1731616204417 WAL count=1, meta=false 2024-11-14T20:30:29,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 6c8768110af2%2C42163%2C1731616204417.1731616206877}] 2024-11-14T20:30:30,005 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=6c8768110af2,44657,1731616204351 2024-11-14T20:30:30,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 6c8768110af2%2C42163%2C1731616204417.1731616206877, worker=6c8768110af2,44657,1731616204351}] 2024-11-14T20:30:30,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44657 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-14T20:30:30,198 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877, size=1.4 K (1397bytes) 2024-11-14T20:30:30,198 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877 2024-11-14T20:30:30,200 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877 after 2ms 2024-11-14T20:30:30,207 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:30,207 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877 took 9ms 2024-11-14T20:30:30,213 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 18f7838e8e8a18bc6e359dc30256cd47: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-14T20:30:30,213 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877 so closing down 2024-11-14T20:30:30,213 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:30,214 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:30,214 INFO [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877, size=1.4 K, length=1397, corrupted=false, cancelled=false 2024-11-14T20:30:30,215 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877, journal: Splitting hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877, size=1.4 K (1397bytes) at 1731616230198Finishing writing output for hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877 so closing down at 1731616230213 (+15 ms)3 split writer threads finished at 1731616230214 (+1 ms)Processed 6 edits across 0 Regions in 6 ms; skipped=6; WAL=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877, size=1.4 K, length=1397, corrupted=false, cancelled=false at 1731616230214 2024-11-14T20:30:30,215 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877 2024-11-14T20:30:30,215 DEBUG [RS_LOG_REPLAY_OPS-regionserver/6c8768110af2:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-14T20:30:30,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36945 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-14T20:30:30,222 INFO [PEWorker-2 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting/6c8768110af2%2C42163%2C1731616204417.1731616206877 to hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs 2024-11-14T20:30:30,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-14T20:30:30,227 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 6c8768110af2%2C42163%2C1731616204417.1731616206877, worker=6c8768110af2,44657,1731616204351 in 213 msec 2024-11-14T20:30:30,229 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=6c8768110af2,44657,1731616204351 2024-11-14T20:30:30,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-14T20:30:30,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 6c8768110af2%2C42163%2C1731616204417.1731616206877, worker=6c8768110af2,44657,1731616204351 in 230 msec 2024-11-14T20:30:30,236 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,42163,1731616204417-splitting dir is empty, no logs to split. 2024-11-14T20:30:30,236 INFO [PEWorker-5 {}] master.SplitWALManager(105): 6c8768110af2,42163,1731616204417 WAL count=0, meta=false 2024-11-14T20:30:30,236 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if 6c8768110af2,42163,1731616204417 WAL splitting is done? wals=0, meta=false 2024-11-14T20:30:30,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN}] 2024-11-14T20:30:30,240 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN 2024-11-14T20:30:30,241 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-14T20:30:30,392 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(204): Hosts are {6c8768110af2=0} racks are {/default-rack=0} 2024-11-14T20:30:30,392 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-14T20:30:30,392 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-14T20:30:30,393 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-14T20:30:30,393 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-14T20:30:30,393 INFO [6c8768110af2:36945 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-14T20:30:30,393 INFO [6c8768110af2:36945 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-14T20:30:30,393 DEBUG [6c8768110af2:36945 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-14T20:30:30,393 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPENING, regionLocation=6c8768110af2,46509,1731616204216 2024-11-14T20:30:30,395 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6c8768110af2:42163 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6c8768110af2/172.17.0.3:42163 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:30,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1, error=java.net.ConnectException: Call to address=6c8768110af2:42163 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6c8768110af2/172.17.0.3:42163 2024-11-14T20:30:30,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-14T20:30:30,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6c8768110af2,42163,1731616204417, seqNum=-1 from cache 2024-11-14T20:30:30,396 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 6c8768110af2:42163 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6c8768110af2/172.17.0.3:42163 2024-11-14T20:30:30,499 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:30,499 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,46509,1731616204216, seqNum=-1] 2024-11-14T20:30:30,500 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-14T20:30:30,501 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55477, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-14T20:30:30,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN because future has completed 2024-11-14T20:30:30,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216}] 2024-11-14T20:30:30,662 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:30,663 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 18f7838e8e8a18bc6e359dc30256cd47, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:30,663 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,663 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:30,664 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,664 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,665 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,666 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf1 2024-11-14T20:30:30,667 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:30,674 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf1/b3026c2cbb3a42c7870a1fec4bec7f98 2024-11-14T20:30:30,675 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:30,675 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,676 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18f7838e8e8a18bc6e359dc30256cd47 columnFamilyName cf2 2024-11-14T20:30:30,676 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:30,682 DEBUG [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/cf2/bc12c42e80b146359be4bf6eeffdb370 2024-11-14T20:30:30,682 INFO [StoreOpener-18f7838e8e8a18bc6e359dc30256cd47-1 {}] regionserver.HStore(327): Store=18f7838e8e8a18bc6e359dc30256cd47/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:30,682 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,683 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,684 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,685 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,685 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,686 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-14T20:30:30,687 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,688 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 18f7838e8e8a18bc6e359dc30256cd47; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61270189, jitterRate=-0.08700303733348846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-14T20:30:30,688 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:30,689 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 18f7838e8e8a18bc6e359dc30256cd47: Running coprocessor pre-open hook at 1731616230664Writing region info on filesystem at 1731616230664Initializing all the Stores at 1731616230665 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616230665Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616230665Cleaning up temporary data from old regions at 1731616230685 (+20 ms)Running coprocessor post-open hooks at 1731616230688 (+3 ms)Region opened successfully at 1731616230689 (+1 ms) 2024-11-14T20:30:30,690 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., pid=21, masterSystemTime=1731616230658 2024-11-14T20:30:30,693 DEBUG [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:30,693 INFO [RS_OPEN_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:30,694 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=18f7838e8e8a18bc6e359dc30256cd47, regionState=OPEN, openSeqNum=18, regionLocation=6c8768110af2,46509,1731616204216 2024-11-14T20:30:30,697 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216 because future has completed 2024-11-14T20:30:30,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-14T20:30:30,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 18f7838e8e8a18bc6e359dc30256cd47, server=6c8768110af2,46509,1731616204216 in 194 msec 2024-11-14T20:30:30,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-14T20:30:30,712 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 6c8768110af2,42163,1731616204417 after splitting done 2024-11-14T20:30:30,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=18f7838e8e8a18bc6e359dc30256cd47, ASSIGN in 470 msec 2024-11-14T20:30:30,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 6c8768110af2,42163,1731616204417, splitWal=true, meta=true in 1.6730 sec 2024-11-14T20:30:30,839 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-14T20:30:30,841 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6c8768110af2,46509,1731616204216, seqNum=-1] 2024-11-14T20:30:30,960 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 6c8768110af2:42163 this server is in the failed servers list 2024-11-14T20:30:30,960 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=6c8768110af2:42163 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6c8768110af2:42163 2024-11-14T20:30:30,960 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 6c8768110af2:42163 2024-11-14T20:30:30,960 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,42163,1731616204417, seqNum=12 from cache 2024-11-14T20:30:31,177 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47., hostname=6c8768110af2,46509,1731616204216, seqNum=18] 2024-11-14T20:30:31,205 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=402 (was 409), OpenFileDescriptor=1040 (was 1022) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=692 (was 683) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3712 (was 4109) 2024-11-14T20:30:31,207 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-14T20:30:31,230 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=402, OpenFileDescriptor=1040, MaxFileDescriptor=1048576, SystemLoadAverage=692, ProcessCount=11, AvailableMemoryMB=3710 2024-11-14T20:30:31,230 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1040 is superior to 1024 2024-11-14T20:30:31,250 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:31,252 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:31,253 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:31,255 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23092838, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-23092838, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:31,270 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23092838/hregion-23092838.1731616231256, exclude list is [], retry=0 2024-11-14T20:30:31,273 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:31,277 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:31,281 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:31,297 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23092838/hregion-23092838.1731616231256 2024-11-14T20:30:31,301 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:31,301 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 54e3b6707238c10c8c126c1038b1da5b, NAME => 'testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:31,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741898_1076 (size=67) 2024-11-14T20:30:31,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741898_1076 (size=67) 2024-11-14T20:30:31,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741898_1076 (size=67) 2024-11-14T20:30:31,313 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:31,315 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,317 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName a 2024-11-14T20:30:31,317 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:31,318 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:31,318 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,320 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName b 2024-11-14T20:30:31,320 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:31,321 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:31,321 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,323 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName c 2024-11-14T20:30:31,323 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:31,324 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:31,324 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,325 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,326 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,328 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,328 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,329 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:31,331 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,333 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:31,335 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 54e3b6707238c10c8c126c1038b1da5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59457937, jitterRate=-0.11400769650936127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:31,335 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 54e3b6707238c10c8c126c1038b1da5b: Writing region info on filesystem at 1731616231313Initializing all the Stores at 1731616231314 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616231314Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616231315 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616231315Cleaning up temporary data from old regions at 1731616231328 (+13 ms)Region opened successfully at 1731616231335 (+7 ms) 2024-11-14T20:30:31,335 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 54e3b6707238c10c8c126c1038b1da5b, disabling compactions & flushes 2024-11-14T20:30:31,335 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:31,335 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:31,336 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. after waiting 0 ms 2024-11-14T20:30:31,336 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:31,336 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:31,336 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 54e3b6707238c10c8c126c1038b1da5b: Waiting for close lock at 1731616231335Disabling compacts and flushes for region at 1731616231335Disabling writes for close at 1731616231336 (+1 ms)Writing region close event to WAL at 1731616231336Closed at 1731616231336 2024-11-14T20:30:31,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741897_1075 (size=95) 2024-11-14T20:30:31,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741897_1075 (size=95) 2024-11-14T20:30:31,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741897_1075 (size=95) 2024-11-14T20:30:31,349 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:31,349 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23092838:(num 1731616231256) 2024-11-14T20:30:31,350 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:31,352 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:31,373 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352, exclude list is [], retry=0 2024-11-14T20:30:31,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:31,377 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:31,378 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:31,405 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352 2024-11-14T20:30:31,405 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:31,405 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 54e3b6707238c10c8c126c1038b1da5b, NAME => 'testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:31,406 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:31,406 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,406 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,409 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,410 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName a 2024-11-14T20:30:31,410 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:31,411 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:31,411 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,412 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName b 2024-11-14T20:30:31,412 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:31,416 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:31,417 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,419 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName c 2024-11-14T20:30:31,419 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:31,420 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:31,421 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,423 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,425 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,427 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,427 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,428 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:31,430 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:31,432 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 54e3b6707238c10c8c126c1038b1da5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64643386, jitterRate=-0.03673848509788513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:31,433 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 54e3b6707238c10c8c126c1038b1da5b: Writing region info on filesystem at 1731616231406Initializing all the Stores at 1731616231407 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616231407Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616231408 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616231408Cleaning up temporary data from old regions at 1731616231427 (+19 ms)Region opened successfully at 1731616231433 (+6 ms) 2024-11-14T20:30:31,506 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 54e3b6707238c10c8c126c1038b1da5b 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T20:30:31,551 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/a/0bbe2238be1245efaa2eb591a09229c8 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731616231433/Put/seqid=0 2024-11-14T20:30:31,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741900_1078 (size=5958) 2024-11-14T20:30:31,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741900_1078 (size=5958) 2024-11-14T20:30:31,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741900_1078 (size=5958) 2024-11-14T20:30:31,613 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/a/0bbe2238be1245efaa2eb591a09229c8 2024-11-14T20:30:31,665 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/b/e140eb1101194a22a33476c4acdfecc8 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731616231453/Put/seqid=0 2024-11-14T20:30:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741901_1079 (size=5958) 2024-11-14T20:30:31,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741901_1079 (size=5958) 2024-11-14T20:30:31,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741901_1079 (size=5958) 2024-11-14T20:30:31,714 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/b/e140eb1101194a22a33476c4acdfecc8 2024-11-14T20:30:31,735 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T20:30:31,753 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/c/adbeaa972f45479498985db4c09778da is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731616231469/Put/seqid=0 2024-11-14T20:30:31,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741902_1080 (size=5958) 2024-11-14T20:30:31,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741902_1080 (size=5958) 2024-11-14T20:30:31,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741902_1080 (size=5958) 2024-11-14T20:30:31,833 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/c/adbeaa972f45479498985db4c09778da 2024-11-14T20:30:31,849 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/a/0bbe2238be1245efaa2eb591a09229c8 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/a/0bbe2238be1245efaa2eb591a09229c8 2024-11-14T20:30:31,887 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/a/0bbe2238be1245efaa2eb591a09229c8, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T20:30:31,891 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/b/e140eb1101194a22a33476c4acdfecc8 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/b/e140eb1101194a22a33476c4acdfecc8 2024-11-14T20:30:31,942 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/b/e140eb1101194a22a33476c4acdfecc8, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T20:30:31,952 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/c/adbeaa972f45479498985db4c09778da as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/c/adbeaa972f45479498985db4c09778da 2024-11-14T20:30:32,001 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/c/adbeaa972f45479498985db4c09778da, entries=10, sequenceid=33, filesize=5.8 K 2024-11-14T20:30:32,005 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 54e3b6707238c10c8c126c1038b1da5b in 498ms, sequenceid=33, compaction requested=false 2024-11-14T20:30:32,005 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 54e3b6707238c10c8c126c1038b1da5b: 2024-11-14T20:30:32,005 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 54e3b6707238c10c8c126c1038b1da5b, disabling compactions & flushes 2024-11-14T20:30:32,005 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:32,005 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:32,005 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. after waiting 0 ms 2024-11-14T20:30:32,005 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:32,049 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. 2024-11-14T20:30:32,049 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 54e3b6707238c10c8c126c1038b1da5b: Waiting for close lock at 1731616232005Disabling compacts and flushes for region at 1731616232005Disabling writes for close at 1731616232005Writing region close event to WAL at 1731616232049 (+44 ms)Closed at 1731616232049 2024-11-14T20:30:32,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741899_1077 (size=3382) 2024-11-14T20:30:32,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741899_1077 (size=3382) 2024-11-14T20:30:32,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741899_1077 (size=3382) 2024-11-14T20:30:32,113 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/b/e140eb1101194a22a33476c4acdfecc8 to hdfs://localhost:43677/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/b/e140eb1101194a22a33476c4acdfecc8 2024-11-14T20:30:32,146 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352, size=3.3 K (3382bytes) 2024-11-14T20:30:32,146 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352 2024-11-14T20:30:32,147 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352 after 0ms 2024-11-14T20:30:32,153 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:32,153 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352 took 7ms 2024-11-14T20:30:32,164 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352 so closing down 2024-11-14T20:30:32,164 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:32,167 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731616231352.temp 2024-11-14T20:30:32,169 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000003-wal.1731616231352.temp 2024-11-14T20:30:32,170 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741903_1081 (size=2944) 2024-11-14T20:30:32,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741903_1081 (size=2944) 2024-11-14T20:30:32,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741903_1081 (size=2944) 2024-11-14T20:30:32,249 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-14T20:30:32,608 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000003-wal.1731616231352.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:32,610 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000003-wal.1731616231352.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000032 2024-11-14T20:30:32,611 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 449 ms; skipped=2; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352, size=3.3 K, length=3382, corrupted=false, cancelled=false 2024-11-14T20:30:32,611 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352, journal: Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352, size=3.3 K (3382bytes) at 1731616232146Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352 so closing down at 1731616232164 (+18 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000003-wal.1731616231352.temp at 1731616232169 (+5 ms)3 split writer threads finished at 1731616232172 (+3 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000003-wal.1731616231352.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731616232608 (+436 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000003-wal.1731616231352.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000032 at 1731616232610 (+2 ms)Processed 32 edits across 1 Regions in 449 ms; skipped=2; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352, size=3.3 K, length=3382, corrupted=false, cancelled=false at 1731616232611 (+1 ms) 2024-11-14T20:30:32,613 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616231352 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616231352 2024-11-14T20:30:32,614 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000032 2024-11-14T20:30:32,614 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:32,616 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:32,631 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616232616, exclude list is [], retry=0 2024-11-14T20:30:32,634 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:32,634 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:32,634 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:32,644 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616232616 2024-11-14T20:30:32,644 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:32,644 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 54e3b6707238c10c8c126c1038b1da5b, NAME => 'testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:32,644 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:32,645 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,645 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,648 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,649 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName a 2024-11-14T20:30:32,649 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:32,658 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/a/0bbe2238be1245efaa2eb591a09229c8 2024-11-14T20:30:32,658 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:32,658 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,659 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName b 2024-11-14T20:30:32,659 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:32,660 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:32,660 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,661 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 54e3b6707238c10c8c126c1038b1da5b columnFamilyName c 2024-11-14T20:30:32,661 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:32,669 DEBUG [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/c/adbeaa972f45479498985db4c09778da 2024-11-14T20:30:32,669 INFO [StoreOpener-54e3b6707238c10c8c126c1038b1da5b-1 {}] regionserver.HStore(327): Store=54e3b6707238c10c8c126c1038b1da5b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:32,669 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,670 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,671 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,672 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000032 2024-11-14T20:30:32,673 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:32,675 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000032 2024-11-14T20:30:32,675 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 54e3b6707238c10c8c126c1038b1da5b 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T20:30:32,690 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/b/a71576b3089e453ab5a5ae369c30b7ea is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731616231453/Put/seqid=0 2024-11-14T20:30:32,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741905_1083 (size=5958) 2024-11-14T20:30:32,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741905_1083 (size=5958) 2024-11-14T20:30:32,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741905_1083 (size=5958) 2024-11-14T20:30:32,703 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/b/a71576b3089e453ab5a5ae369c30b7ea 2024-11-14T20:30:32,710 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/.tmp/b/a71576b3089e453ab5a5ae369c30b7ea as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/b/a71576b3089e453ab5a5ae369c30b7ea 2024-11-14T20:30:32,715 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/b/a71576b3089e453ab5a5ae369c30b7ea, entries=10, sequenceid=32, filesize=5.8 K 2024-11-14T20:30:32,716 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 54e3b6707238c10c8c126c1038b1da5b in 40ms, sequenceid=32, compaction requested=false; wal=null 2024-11-14T20:30:32,716 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/0000000000000000032 2024-11-14T20:30:32,718 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,718 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,718 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:32,721 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 54e3b6707238c10c8c126c1038b1da5b 2024-11-14T20:30:32,723 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/54e3b6707238c10c8c126c1038b1da5b/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-14T20:30:32,724 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 54e3b6707238c10c8c126c1038b1da5b; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64082696, jitterRate=-0.045093417167663574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:32,725 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 54e3b6707238c10c8c126c1038b1da5b: Writing region info on filesystem at 1731616232645Initializing all the Stores at 1731616232646 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616232646Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616232648 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616232648Obtaining lock to block concurrent updates at 1731616232675 (+27 ms)Preparing flush snapshotting stores in 54e3b6707238c10c8c126c1038b1da5b at 1731616232675Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731616232675Flushing stores of testReplayEditsWrittenViaHRegion,,1731616231251.54e3b6707238c10c8c126c1038b1da5b. at 1731616232675Flushing 54e3b6707238c10c8c126c1038b1da5b/b: creating writer at 1731616232675Flushing 54e3b6707238c10c8c126c1038b1da5b/b: appending metadata at 1731616232689 (+14 ms)Flushing 54e3b6707238c10c8c126c1038b1da5b/b: closing flushed file at 1731616232689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4dceec: reopening flushed file at 1731616232709 (+20 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 54e3b6707238c10c8c126c1038b1da5b in 40ms, sequenceid=32, compaction requested=false; wal=null at 1731616232716 (+7 ms)Cleaning up temporary data from old regions at 1731616232718 (+2 ms)Region opened successfully at 1731616232725 (+7 ms) 2024-11-14T20:30:32,758 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=412 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:42128 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:36178 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:41988 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:41128 [Waiting for operation #50] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:41280 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:36386 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1114 (was 1040) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=757 (was 692) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3660 (was 3710) 2024-11-14T20:30:32,758 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-11-14T20:30:32,772 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=412, OpenFileDescriptor=1114, MaxFileDescriptor=1048576, SystemLoadAverage=757, ProcessCount=11, AvailableMemoryMB=3658 2024-11-14T20:30:32,772 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1114 is superior to 1024 2024-11-14T20:30:32,789 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:32,791 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:32,792 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:32,794 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-74477073, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-74477073, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:32,807 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-74477073/hregion-74477073.1731616232795, exclude list is [], retry=0 2024-11-14T20:30:32,809 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:32,810 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:32,811 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:32,817 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-74477073/hregion-74477073.1731616232795 2024-11-14T20:30:32,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:32,817 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8d47b95cb4a43faaf94d42dde49e963a, NAME => 'testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:32,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741907_1085 (size=68) 2024-11-14T20:30:32,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741907_1085 (size=68) 2024-11-14T20:30:32,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741907_1085 (size=68) 2024-11-14T20:30:32,838 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:32,840 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,842 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName a 2024-11-14T20:30:32,842 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:32,843 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:32,843 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,844 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName b 2024-11-14T20:30:32,844 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:32,845 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:32,845 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,847 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName c 2024-11-14T20:30:32,847 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:32,847 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:32,847 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,848 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,848 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,849 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,849 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,850 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:32,852 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:32,860 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:32,862 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d47b95cb4a43faaf94d42dde49e963a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60479334, jitterRate=-0.09878769516944885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:32,862 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d47b95cb4a43faaf94d42dde49e963a: Writing region info on filesystem at 1731616232838Initializing all the Stores at 1731616232839 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616232839Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616232840 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616232840Cleaning up temporary data from old regions at 1731616232849 (+9 ms)Region opened successfully at 1731616232862 (+13 ms) 2024-11-14T20:30:32,863 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d47b95cb4a43faaf94d42dde49e963a, disabling compactions & flushes 2024-11-14T20:30:32,863 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:32,863 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:32,863 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. after waiting 0 ms 2024-11-14T20:30:32,863 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:32,873 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:32,873 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d47b95cb4a43faaf94d42dde49e963a: Waiting for close lock at 1731616232862Disabling compacts and flushes for region at 1731616232863 (+1 ms)Disabling writes for close at 1731616232863Writing region close event to WAL at 1731616232873 (+10 ms)Closed at 1731616232873 2024-11-14T20:30:32,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741906_1084 (size=95) 2024-11-14T20:30:32,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741906_1084 (size=95) 2024-11-14T20:30:32,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741906_1084 (size=95) 2024-11-14T20:30:32,893 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:32,893 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-74477073:(num 1731616232795) 2024-11-14T20:30:32,893 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:32,895 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:32,916 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895, exclude list is [], retry=0 2024-11-14T20:30:32,922 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:32,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:32,923 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:32,932 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895 2024-11-14T20:30:32,933 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:33,021 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d47b95cb4a43faaf94d42dde49e963a, NAME => 'testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:33,024 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,024 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:33,025 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,025 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,027 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,029 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName a 2024-11-14T20:30:33,029 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:33,030 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:33,030 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,031 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName b 2024-11-14T20:30:33,031 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:33,032 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:33,032 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,033 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName c 2024-11-14T20:30:33,033 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:33,033 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:33,034 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,034 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,036 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,038 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,038 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,038 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:33,040 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,041 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d47b95cb4a43faaf94d42dde49e963a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59509694, jitterRate=-0.1132364571094513}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:33,041 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:33,042 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d47b95cb4a43faaf94d42dde49e963a: Running coprocessor pre-open hook at 1731616233025Writing region info on filesystem at 1731616233025Initializing all the Stores at 1731616233026 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616233026Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616233027 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616233027Cleaning up temporary data from old regions at 1731616233038 (+11 ms)Running coprocessor post-open hooks at 1731616233041 (+3 ms)Region opened successfully at 1731616233042 (+1 ms) 2024-11-14T20:30:33,087 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d47b95cb4a43faaf94d42dde49e963a 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-14T20:30:33,092 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:33,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-14T20:30:33,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:33,836 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-14T20:30:33,836 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:33,837 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-14T20:30:33,837 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:34,093 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:35,093 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:36,094 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:37,095 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:38,095 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:38,527 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T20:30:39,096 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:40,097 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:41,098 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:42,099 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 8d47b95cb4a43faaf94d42dde49e963a/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:42,100 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8d47b95cb4a43faaf94d42dde49e963a: 2024-11-14T20:30:42,100 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:42,136 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8d47b95cb4a43faaf94d42dde49e963a: 2024-11-14T20:30:42,136 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-14T20:30:42,136 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8d47b95cb4a43faaf94d42dde49e963a, disabling compactions & flushes 2024-11-14T20:30:42,136 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:42,136 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:42,136 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. after waiting 0 ms 2024-11-14T20:30:42,136 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:42,139 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:42,140 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. 2024-11-14T20:30:42,140 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8d47b95cb4a43faaf94d42dde49e963a: Waiting for close lock at 1731616242136Running coprocessor pre-close hooks at 1731616242136Disabling compacts and flushes for region at 1731616242136Disabling writes for close at 1731616242136Writing region close event to WAL at 1731616242139 (+3 ms)Running coprocessor post-close hooks at 1731616242139Closed at 1731616242140 (+1 ms) 2024-11-14T20:30:42,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741908_1086 (size=2691) 2024-11-14T20:30:42,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741908_1086 (size=2691) 2024-11-14T20:30:42,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741908_1086 (size=2691) 2024-11-14T20:30:42,182 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895, size=2.6 K (2691bytes) 2024-11-14T20:30:42,182 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895 2024-11-14T20:30:42,183 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895 after 1ms 2024-11-14T20:30:42,187 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:42,187 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895 took 5ms 2024-11-14T20:30:42,195 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895 so closing down 2024-11-14T20:30:42,195 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:42,203 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731616232895.temp 2024-11-14T20:30:42,207 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000004-wal.1731616232895.temp 2024-11-14T20:30:42,207 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:42,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741909_1087 (size=2094) 2024-11-14T20:30:42,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741909_1087 (size=2094) 2024-11-14T20:30:42,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741909_1087 (size=2094) 2024-11-14T20:30:42,243 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000004-wal.1731616232895.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:42,250 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000004-wal.1731616232895.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000026 2024-11-14T20:30:42,252 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 60 ms; skipped=3; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-11-14T20:30:42,265 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895, journal: Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895, size=2.6 K (2691bytes) at 1731616242182Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895 so closing down at 1731616242195 (+13 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000004-wal.1731616232895.temp at 1731616242207 (+12 ms)3 split writer threads finished at 1731616242207Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000004-wal.1731616232895.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731616242243 (+36 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000004-wal.1731616232895.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000026 at 1731616242250 (+7 ms)Processed 23 edits across 1 Regions in 60 ms; skipped=3; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1731616242264 (+14 ms) 2024-11-14T20:30:42,274 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616232895 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616232895 2024-11-14T20:30:42,277 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000026 2024-11-14T20:30:42,277 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:42,294 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:42,316 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616242295, exclude list is [], retry=0 2024-11-14T20:30:42,333 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:42,337 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:42,341 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:42,374 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616242295 2024-11-14T20:30:42,375 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:42,375 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8d47b95cb4a43faaf94d42dde49e963a, NAME => 'testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:42,376 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,376 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:42,376 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,376 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,388 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,389 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName a 2024-11-14T20:30:42,389 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:42,390 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:42,391 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,392 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName b 2024-11-14T20:30:42,392 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:42,393 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:42,393 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,395 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8d47b95cb4a43faaf94d42dde49e963a columnFamilyName c 2024-11-14T20:30:42,395 DEBUG [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:42,399 INFO [StoreOpener-8d47b95cb4a43faaf94d42dde49e963a-1 {}] regionserver.HStore(327): Store=8d47b95cb4a43faaf94d42dde49e963a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:42,399 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,401 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,409 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,409 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000026 2024-11-14T20:30:42,412 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:42,413 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000026 2024-11-14T20:30:42,414 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8d47b95cb4a43faaf94d42dde49e963a 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-14T20:30:42,441 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/a/82ce947264b2407ca46215264aeb6f09 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731616242106/Put/seqid=0 2024-11-14T20:30:42,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741911_1089 (size=5523) 2024-11-14T20:30:42,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741911_1089 (size=5523) 2024-11-14T20:30:42,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741911_1089 (size=5523) 2024-11-14T20:30:42,479 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/a/82ce947264b2407ca46215264aeb6f09 2024-11-14T20:30:42,502 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/b/94f3a7b3a295419495ea0bb13f23121a is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731616242101/Put/seqid=0 2024-11-14T20:30:42,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741912_1090 (size=5524) 2024-11-14T20:30:42,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741912_1090 (size=5524) 2024-11-14T20:30:42,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741912_1090 (size=5524) 2024-11-14T20:30:42,509 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/b/94f3a7b3a295419495ea0bb13f23121a 2024-11-14T20:30:42,530 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/c/9d97e69a190241b1993dcff9fc44011a is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731616242103/Put/seqid=0 2024-11-14T20:30:42,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741913_1091 (size=5457) 2024-11-14T20:30:42,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741913_1091 (size=5457) 2024-11-14T20:30:42,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741913_1091 (size=5457) 2024-11-14T20:30:42,552 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/c/9d97e69a190241b1993dcff9fc44011a 2024-11-14T20:30:42,562 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/a/82ce947264b2407ca46215264aeb6f09 as hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/a/82ce947264b2407ca46215264aeb6f09 2024-11-14T20:30:42,571 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/a/82ce947264b2407ca46215264aeb6f09, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T20:30:42,572 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/b/94f3a7b3a295419495ea0bb13f23121a as hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/b/94f3a7b3a295419495ea0bb13f23121a 2024-11-14T20:30:42,583 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/b/94f3a7b3a295419495ea0bb13f23121a, entries=7, sequenceid=26, filesize=5.4 K 2024-11-14T20:30:42,584 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/.tmp/c/9d97e69a190241b1993dcff9fc44011a as hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/c/9d97e69a190241b1993dcff9fc44011a 2024-11-14T20:30:42,594 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/c/9d97e69a190241b1993dcff9fc44011a, entries=6, sequenceid=26, filesize=5.3 K 2024-11-14T20:30:42,594 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 8d47b95cb4a43faaf94d42dde49e963a in 180ms, sequenceid=26, compaction requested=false; wal=null 2024-11-14T20:30:42,599 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/0000000000000000026 2024-11-14T20:30:42,601 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,601 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,605 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:42,606 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,609 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsAfterAbortingFlush/8d47b95cb4a43faaf94d42dde49e963a/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-14T20:30:42,610 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8d47b95cb4a43faaf94d42dde49e963a; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72705296, jitterRate=0.08339333534240723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:42,610 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8d47b95cb4a43faaf94d42dde49e963a 2024-11-14T20:30:42,611 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8d47b95cb4a43faaf94d42dde49e963a: Running coprocessor pre-open hook at 1731616242376Writing region info on filesystem at 1731616242376Initializing all the Stores at 1731616242383 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616242383Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616242387 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616242387Obtaining lock to block concurrent updates at 1731616242414 (+27 ms)Preparing flush snapshotting stores in 8d47b95cb4a43faaf94d42dde49e963a at 1731616242414Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731616242416 (+2 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1731616232790.8d47b95cb4a43faaf94d42dde49e963a. at 1731616242416Flushing 8d47b95cb4a43faaf94d42dde49e963a/a: creating writer at 1731616242417 (+1 ms)Flushing 8d47b95cb4a43faaf94d42dde49e963a/a: appending metadata at 1731616242441 (+24 ms)Flushing 8d47b95cb4a43faaf94d42dde49e963a/a: closing flushed file at 1731616242441Flushing 8d47b95cb4a43faaf94d42dde49e963a/b: creating writer at 1731616242486 (+45 ms)Flushing 8d47b95cb4a43faaf94d42dde49e963a/b: appending metadata at 1731616242501 (+15 ms)Flushing 8d47b95cb4a43faaf94d42dde49e963a/b: closing flushed file at 1731616242501Flushing 8d47b95cb4a43faaf94d42dde49e963a/c: creating writer at 1731616242514 (+13 ms)Flushing 8d47b95cb4a43faaf94d42dde49e963a/c: appending metadata at 1731616242530 (+16 ms)Flushing 8d47b95cb4a43faaf94d42dde49e963a/c: closing flushed file at 1731616242530Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b9546ed: reopening flushed file at 1731616242561 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d7a377b: reopening flushed file at 1731616242571 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@493e2d3: reopening flushed file at 1731616242583 (+12 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 8d47b95cb4a43faaf94d42dde49e963a in 180ms, sequenceid=26, compaction requested=false; wal=null at 1731616242594 (+11 ms)Cleaning up temporary data from old regions at 1731616242601 (+7 ms)Running coprocessor post-open hooks at 1731616242610 (+9 ms)Region opened successfully at 1731616242611 (+1 ms) 2024-11-14T20:30:42,641 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=417 (was 412) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:40828 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:60108 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:60118 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48778 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48834 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:40838 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1184 (was 1114) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=749 (was 757), ProcessCount=11 (was 11), AvailableMemoryMB=4794 (was 3658) - AvailableMemoryMB LEAK? - 2024-11-14T20:30:42,641 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-14T20:30:42,655 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1184, MaxFileDescriptor=1048576, SystemLoadAverage=749, ProcessCount=11, AvailableMemoryMB=4791 2024-11-14T20:30:42,655 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1184 is superior to 1024 2024-11-14T20:30:42,677 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:42,679 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:42,680 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:42,687 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-01331201, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-01331201, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:42,708 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-01331201/hregion-01331201.1731616242688, exclude list is [], retry=0 2024-11-14T20:30:42,728 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:42,729 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:42,730 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:42,757 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-01331201/hregion-01331201.1731616242688 2024-11-14T20:30:42,762 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:42,762 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3a7480d18f65ecb138f6081bd570c0c2, NAME => 'testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:42,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741915_1093 (size=61) 2024-11-14T20:30:42,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741915_1093 (size=61) 2024-11-14T20:30:42,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741915_1093 (size=61) 2024-11-14T20:30:43,213 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:43,215 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,218 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a7480d18f65ecb138f6081bd570c0c2 columnFamilyName a 2024-11-14T20:30:43,218 DEBUG [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:43,218 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(327): Store=3a7480d18f65ecb138f6081bd570c0c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:43,219 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,220 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,220 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,221 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,223 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,225 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:43,226 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3a7480d18f65ecb138f6081bd570c0c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66774238, jitterRate=-0.004986315965652466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:43,227 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3a7480d18f65ecb138f6081bd570c0c2: Writing region info on filesystem at 1731616243214Initializing all the Stores at 1731616243215 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616243215Cleaning up temporary data from old regions at 1731616243221 (+6 ms)Region opened successfully at 1731616243227 (+6 ms) 2024-11-14T20:30:43,227 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3a7480d18f65ecb138f6081bd570c0c2, disabling compactions & flushes 2024-11-14T20:30:43,227 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,227 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,227 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. after waiting 0 ms 2024-11-14T20:30:43,227 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,228 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,228 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3a7480d18f65ecb138f6081bd570c0c2: Waiting for close lock at 1731616243227Disabling compacts and flushes for region at 1731616243227Disabling writes for close at 1731616243227Writing region close event to WAL at 1731616243228 (+1 ms)Closed at 1731616243228 2024-11-14T20:30:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741914_1092 (size=95) 2024-11-14T20:30:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741914_1092 (size=95) 2024-11-14T20:30:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741914_1092 (size=95) 2024-11-14T20:30:43,243 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:43,243 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-01331201:(num 1731616242688) 2024-11-14T20:30:43,244 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:43,247 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:43,268 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248, exclude list is [], retry=0 2024-11-14T20:30:43,272 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:43,272 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:43,273 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:43,292 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 2024-11-14T20:30:43,300 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:43,301 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a7480d18f65ecb138f6081bd570c0c2, NAME => 'testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:43,301 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:43,301 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,301 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,313 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,314 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a7480d18f65ecb138f6081bd570c0c2 columnFamilyName a 2024-11-14T20:30:43,314 DEBUG [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:43,315 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(327): Store=3a7480d18f65ecb138f6081bd570c0c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:43,315 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,316 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,317 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,317 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,318 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,319 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:43,320 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3a7480d18f65ecb138f6081bd570c0c2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73744624, jitterRate=0.09888052940368652}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:43,321 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3a7480d18f65ecb138f6081bd570c0c2: Writing region info on filesystem at 1731616243301Initializing all the Stores at 1731616243311 (+10 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616243311Cleaning up temporary data from old regions at 1731616243318 (+7 ms)Region opened successfully at 1731616243321 (+3 ms) 2024-11-14T20:30:43,351 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3a7480d18f65ecb138f6081bd570c0c2, disabling compactions & flushes 2024-11-14T20:30:43,351 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,351 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,351 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. after waiting 0 ms 2024-11-14T20:30:43,351 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,353 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,353 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. 2024-11-14T20:30:43,353 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3a7480d18f65ecb138f6081bd570c0c2: Waiting for close lock at 1731616243351Disabling compacts and flushes for region at 1731616243351Disabling writes for close at 1731616243351Writing region close event to WAL at 1731616243353 (+2 ms)Closed at 1731616243353 2024-11-14T20:30:43,364 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 not finished, retry = 0 2024-11-14T20:30:43,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741916_1094 (size=1050) 2024-11-14T20:30:43,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741916_1094 (size=1050) 2024-11-14T20:30:43,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741916_1094 (size=1050) 2024-11-14T20:30:43,496 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248, size=1.0 K (1050bytes) 2024-11-14T20:30:43,496 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 2024-11-14T20:30:43,497 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 after 1ms 2024-11-14T20:30:43,500 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:43,500 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 took 4ms 2024-11-14T20:30:43,521 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 so closing down 2024-11-14T20:30:43,522 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:43,533 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731616243248.temp 2024-11-14T20:30:43,536 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000003-wal.1731616243248.temp 2024-11-14T20:30:43,537 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741917_1095 (size=1050) 2024-11-14T20:30:43,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741917_1095 (size=1050) 2024-11-14T20:30:43,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741917_1095 (size=1050) 2024-11-14T20:30:43,600 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000003-wal.1731616243248.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:43,613 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000003-wal.1731616243248.temp to hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012 2024-11-14T20:30:43,613 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-11-14T20:30:43,613 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248, journal: Splitting hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248, size=1.0 K (1050bytes) at 1731616243496Finishing writing output for hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 so closing down at 1731616243521 (+25 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000003-wal.1731616243248.temp at 1731616243536 (+15 ms)3 split writer threads finished at 1731616243537 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000003-wal.1731616243248.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731616243601 (+64 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000003-wal.1731616243248.temp to hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012 at 1731616243613 (+12 ms)Processed 10 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1731616243613 2024-11-14T20:30:43,615 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616243248 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616243248 2024-11-14T20:30:43,621 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012 2024-11-14T20:30:43,625 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:44,494 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:44,518 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:44,541 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616244519, exclude list is [], retry=0 2024-11-14T20:30:44,561 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:44,561 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:44,562 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:44,609 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616244519 2024-11-14T20:30:44,611 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:44,611 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a7480d18f65ecb138f6081bd570c0c2, NAME => 'testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:44,612 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:44,612 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,612 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,639 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,651 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a7480d18f65ecb138f6081bd570c0c2 columnFamilyName a 2024-11-14T20:30:44,651 DEBUG [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:44,653 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(327): Store=3a7480d18f65ecb138f6081bd570c0c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:44,653 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,657 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,672 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,677 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012 2024-11-14T20:30:44,685 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:44,689 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012 2024-11-14T20:30:44,690 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3a7480d18f65ecb138f6081bd570c0c2 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-14T20:30:44,736 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/.tmp/a/88376500a5064710bdd8851067bb0429 is 79, key is testDatalossWhenInputError/a:x0/1731616243321/Put/seqid=0 2024-11-14T20:30:44,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741919_1097 (size=5808) 2024-11-14T20:30:44,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741919_1097 (size=5808) 2024-11-14T20:30:44,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741919_1097 (size=5808) 2024-11-14T20:30:44,808 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/.tmp/a/88376500a5064710bdd8851067bb0429 2024-11-14T20:30:44,868 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/.tmp/a/88376500a5064710bdd8851067bb0429 as hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/a/88376500a5064710bdd8851067bb0429 2024-11-14T20:30:44,909 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/a/88376500a5064710bdd8851067bb0429, entries=10, sequenceid=12, filesize=5.7 K 2024-11-14T20:30:44,910 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 3a7480d18f65ecb138f6081bd570c0c2 in 220ms, sequenceid=12, compaction requested=false; wal=null 2024-11-14T20:30:44,916 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/0000000000000000012 2024-11-14T20:30:44,918 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,918 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,928 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,943 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-14T20:30:44,948 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3a7480d18f65ecb138f6081bd570c0c2; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70400574, jitterRate=0.04905030131340027}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:44,949 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3a7480d18f65ecb138f6081bd570c0c2: Writing region info on filesystem at 1731616244612Initializing all the Stores at 1731616244635 (+23 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616244635Obtaining lock to block concurrent updates at 1731616244690 (+55 ms)Preparing flush snapshotting stores in 3a7480d18f65ecb138f6081bd570c0c2 at 1731616244690Finished memstore snapshotting testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731616244690Flushing stores of testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2. at 1731616244690Flushing 3a7480d18f65ecb138f6081bd570c0c2/a: creating writer at 1731616244690Flushing 3a7480d18f65ecb138f6081bd570c0c2/a: appending metadata at 1731616244735 (+45 ms)Flushing 3a7480d18f65ecb138f6081bd570c0c2/a: closing flushed file at 1731616244735Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bea4358: reopening flushed file at 1731616244854 (+119 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 3a7480d18f65ecb138f6081bd570c0c2 in 220ms, sequenceid=12, compaction requested=false; wal=null at 1731616244910 (+56 ms)Cleaning up temporary data from old regions at 1731616244918 (+8 ms)Region opened successfully at 1731616244949 (+31 ms) 2024-11-14T20:30:44,954 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a7480d18f65ecb138f6081bd570c0c2, NAME => 'testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:44,954 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731616242678.3a7480d18f65ecb138f6081bd570c0c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:44,954 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,954 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,957 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,958 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a7480d18f65ecb138f6081bd570c0c2 columnFamilyName a 2024-11-14T20:30:44,958 DEBUG [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:44,966 DEBUG [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/a/88376500a5064710bdd8851067bb0429 2024-11-14T20:30:44,966 INFO [StoreOpener-3a7480d18f65ecb138f6081bd570c0c2-1 {}] regionserver.HStore(327): Store=3a7480d18f65ecb138f6081bd570c0c2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:44,967 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,968 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,969 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,969 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,971 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3a7480d18f65ecb138f6081bd570c0c2 2024-11-14T20:30:44,973 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testDatalossWhenInputError/3a7480d18f65ecb138f6081bd570c0c2/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-14T20:30:44,974 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3a7480d18f65ecb138f6081bd570c0c2; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65000127, jitterRate=-0.031422629952430725}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-14T20:30:44,974 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3a7480d18f65ecb138f6081bd570c0c2: Writing region info on filesystem at 1731616244954Initializing all the Stores at 1731616244955 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616244955Cleaning up temporary data from old regions at 1731616244969 (+14 ms)Region opened successfully at 1731616244974 (+5 ms) 2024-11-14T20:30:44,998 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=426 (was 417) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:40900 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:60108 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48872 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:60198 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-866344823-172.17.0.3-1731616199461:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:48834 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:40838 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1266 (was 1184) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=749 (was 749), ProcessCount=11 (was 11), AvailableMemoryMB=5297 (was 4791) - AvailableMemoryMB LEAK? - 2024-11-14T20:30:44,998 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-14T20:30:45,015 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=426, OpenFileDescriptor=1266, MaxFileDescriptor=1048576, SystemLoadAverage=749, ProcessCount=11, AvailableMemoryMB=5291 2024-11-14T20:30:45,016 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1266 is superior to 1024 2024-11-14T20:30:45,038 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:45,046 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:45,046 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:45,053 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-78255537, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-78255537, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:45,075 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-78255537/hregion-78255537.1731616245054, exclude list is [], retry=0 2024-11-14T20:30:45,086 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:45,088 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:45,089 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:45,097 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-78255537/hregion-78255537.1731616245054 2024-11-14T20:30:45,100 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289)] 2024-11-14T20:30:45,100 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b2a5ef733f596f2369a9662d21c49807, NAME => 'testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741921_1099 (size=63) 2024-11-14T20:30:45,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741921_1099 (size=63) 2024-11-14T20:30:45,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741921_1099 (size=63) 2024-11-14T20:30:45,131 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:45,137 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,139 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName a 2024-11-14T20:30:45,139 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:45,141 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:45,141 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,142 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName b 2024-11-14T20:30:45,142 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:45,143 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:45,143 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,145 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName c 2024-11-14T20:30:45,145 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:45,145 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:45,145 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,152 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,156 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,172 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,172 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,178 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:45,183 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,189 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:45,191 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b2a5ef733f596f2369a9662d21c49807; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68325643, jitterRate=0.018131420016288757}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:45,192 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b2a5ef733f596f2369a9662d21c49807: Writing region info on filesystem at 1731616245131Initializing all the Stores at 1731616245133 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616245133Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616245136 (+3 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616245136Cleaning up temporary data from old regions at 1731616245172 (+36 ms)Region opened successfully at 1731616245192 (+20 ms) 2024-11-14T20:30:45,192 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b2a5ef733f596f2369a9662d21c49807, disabling compactions & flushes 2024-11-14T20:30:45,192 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:45,192 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:45,192 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. after waiting 0 ms 2024-11-14T20:30:45,192 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:45,195 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:45,195 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b2a5ef733f596f2369a9662d21c49807: Waiting for close lock at 1731616245192Disabling compacts and flushes for region at 1731616245192Disabling writes for close at 1731616245192Writing region close event to WAL at 1731616245194 (+2 ms)Closed at 1731616245194 2024-11-14T20:30:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741920_1098 (size=95) 2024-11-14T20:30:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741920_1098 (size=95) 2024-11-14T20:30:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741920_1098 (size=95) 2024-11-14T20:30:45,215 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:45,215 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-78255537:(num 1731616245054) 2024-11-14T20:30:45,215 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:45,221 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:45,240 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221, exclude list is [], retry=0 2024-11-14T20:30:45,248 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:45,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:45,249 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:45,261 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 2024-11-14T20:30:45,264 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:45,265 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b2a5ef733f596f2369a9662d21c49807, NAME => 'testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:45,265 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:45,265 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,265 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,269 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,270 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName a 2024-11-14T20:30:45,270 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:45,270 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:45,271 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,271 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName b 2024-11-14T20:30:45,271 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:45,272 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:45,272 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,273 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName c 2024-11-14T20:30:45,273 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:45,273 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:45,273 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,274 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,275 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,276 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,276 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,276 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:45,277 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:45,278 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b2a5ef733f596f2369a9662d21c49807; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70116149, jitterRate=0.04481203854084015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:45,279 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b2a5ef733f596f2369a9662d21c49807: Writing region info on filesystem at 1731616245265Initializing all the Stores at 1731616245266 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616245266Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616245268 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616245268Cleaning up temporary data from old regions at 1731616245276 (+8 ms)Region opened successfully at 1731616245279 (+3 ms) 2024-11-14T20:30:45,285 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731616245285/Put/seqid=0 2024-11-14T20:30:45,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741923_1101 (size=4875) 2024-11-14T20:30:45,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741923_1101 (size=4875) 2024-11-14T20:30:45,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741923_1101 (size=4875) 2024-11-14T20:30:45,302 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731616245302/Put/seqid=0 2024-11-14T20:30:45,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741924_1102 (size=4875) 2024-11-14T20:30:45,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741924_1102 (size=4875) 2024-11-14T20:30:45,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741924_1102 (size=4875) 2024-11-14T20:30:45,322 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731616245322/Put/seqid=0 2024-11-14T20:30:45,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741925_1103 (size=4875) 2024-11-14T20:30:45,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741925_1103 (size=4875) 2024-11-14T20:30:45,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741925_1103 (size=4875) 2024-11-14T20:30:45,353 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in b2a5ef733f596f2369a9662d21c49807/a 2024-11-14T20:30:45,362 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-14T20:30:45,362 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T20:30:45,362 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in b2a5ef733f596f2369a9662d21c49807/a 2024-11-14T20:30:45,367 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-14T20:30:45,367 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T20:30:45,367 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in b2a5ef733f596f2369a9662d21c49807/a 2024-11-14T20:30:45,370 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-14T20:30:45,371 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-14T20:30:45,371 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b2a5ef733f596f2369a9662d21c49807 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-14T20:30:45,403 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/.tmp/a/f6963d66ed304f89b0fdee8b726c133b is 55, key is testCompactedBulkLoadedFiles/a:a/1731616245279/Put/seqid=0 2024-11-14T20:30:45,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741926_1104 (size=5107) 2024-11-14T20:30:45,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741926_1104 (size=5107) 2024-11-14T20:30:45,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741926_1104 (size=5107) 2024-11-14T20:30:45,425 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/.tmp/a/f6963d66ed304f89b0fdee8b726c133b 2024-11-14T20:30:45,444 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/.tmp/a/f6963d66ed304f89b0fdee8b726c133b as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b 2024-11-14T20:30:45,456 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b, entries=1, sequenceid=4, filesize=5.0 K 2024-11-14T20:30:45,460 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for b2a5ef733f596f2369a9662d21c49807 in 89ms, sequenceid=4, compaction requested=false 2024-11-14T20:30:45,461 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b2a5ef733f596f2369a9662d21c49807: 2024-11-14T20:30:45,462 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_ 2024-11-14T20:30:45,464 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_ 2024-11-14T20:30:45,469 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_ 2024-11-14T20:30:45,470 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile0 into b2a5ef733f596f2369a9662d21c49807/a as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_ - updating store file list. 2024-11-14T20:30:45,480 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:45,481 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_ into b2a5ef733f596f2369a9662d21c49807/a 2024-11-14T20:30:45,481 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile0 into b2a5ef733f596f2369a9662d21c49807/a (new location: hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_) 2024-11-14T20:30:45,482 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile1 into b2a5ef733f596f2369a9662d21c49807/a as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_ - updating store file list. 2024-11-14T20:30:45,487 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e079a4700fa44456a49d2324861777c7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:45,487 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_ into b2a5ef733f596f2369a9662d21c49807/a 2024-11-14T20:30:45,487 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile1 into b2a5ef733f596f2369a9662d21c49807/a (new location: hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_) 2024-11-14T20:30:45,488 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile2 into b2a5ef733f596f2369a9662d21c49807/a as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_ - updating store file list. 2024-11-14T20:30:45,493 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for fd1b31f0db9a467382d1234cb67664c0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:45,493 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_ into b2a5ef733f596f2369a9662d21c49807/a 2024-11-14T20:30:45,493 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43677/hbase/testCompactedBulkLoadedFiles/hfile2 into b2a5ef733f596f2369a9662d21c49807/a (new location: hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_) 2024-11-14T20:30:45,504 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-14T20:30:45,504 DEBUG [Time-limited test {}] regionserver.HStore(1541): b2a5ef733f596f2369a9662d21c49807/a is initiating major compaction (all files) 2024-11-14T20:30:45,504 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of b2a5ef733f596f2369a9662d21c49807/a in testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:45,504 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_] into tmpdir=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/.tmp, totalSize=19.3 K 2024-11-14T20:30:45,513 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f6963d66ed304f89b0fdee8b726c133b, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731616245279 2024-11-14T20:30:45,515 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T20:30:45,515 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e079a4700fa44456a49d2324861777c7_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T20:30:45,516 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting fd1b31f0db9a467382d1234cb67664c0_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-14T20:30:45,544 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/.tmp/a/67b8f94eec974b48ac20c5563d6c91b6 is 55, key is testCompactedBulkLoadedFiles/a:a/1731616245279/Put/seqid=0 2024-11-14T20:30:45,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741927_1105 (size=6154) 2024-11-14T20:30:45,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741927_1105 (size=6154) 2024-11-14T20:30:45,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741927_1105 (size=6154) 2024-11-14T20:30:45,568 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/.tmp/a/67b8f94eec974b48ac20c5563d6c91b6 as hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/67b8f94eec974b48ac20c5563d6c91b6 2024-11-14T20:30:45,575 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in b2a5ef733f596f2369a9662d21c49807/a of b2a5ef733f596f2369a9662d21c49807 into 67b8f94eec974b48ac20c5563d6c91b6(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-14T20:30:45,576 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for b2a5ef733f596f2369a9662d21c49807: 2024-11-14T20:30:45,576 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T20:30:45,576 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-14T20:30:45,611 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221, size=0 (0bytes) 2024-11-14T20:30:45,612 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 might be still open, length is 0 2024-11-14T20:30:45,612 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 2024-11-14T20:30:45,612 WARN [IPC Server handler 0 on default port 43677 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-14T20:30:45,616 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 after 4ms 2024-11-14T20:30:46,965 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:49664 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41701:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49664 dst: /127.0.0.1:41701 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41701 remote=/127.0.0.1:49664]. Total timeout mills is 60000, 58611 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:46,966 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:38562 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38562 dst: /127.0.0.1:37797 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:46,966 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:56220 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41951:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56220 dst: /127.0.0.1:41951 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741922_1106 (size=1173) 2024-11-14T20:30:46,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741922_1106 (size=1173) 2024-11-14T20:30:46,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741922_1106 (size=1173) 2024-11-14T20:30:47,642 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-14T20:30:47,804 INFO [master/6c8768110af2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-14T20:30:47,804 INFO [master/6c8768110af2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-14T20:30:49,617 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 after 4005ms 2024-11-14T20:30:49,620 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:49,620 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 took 4009ms 2024-11-14T20:30:49,623 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221; continuing. 2024-11-14T20:30:49,623 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 so closing down 2024-11-14T20:30:49,623 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:49,625 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731616245221.temp 2024-11-14T20:30:49,627 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000003-wal.1731616245221.temp 2024-11-14T20:30:49,627 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:49,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741928_1107 (size=547) 2024-11-14T20:30:49,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741928_1107 (size=547) 2024-11-14T20:30:49,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741928_1107 (size=547) 2024-11-14T20:30:49,635 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000003-wal.1731616245221.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:49,637 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000003-wal.1731616245221.temp to hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000008 2024-11-14T20:30:49,637 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T20:30:49,637 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221, journal: Splitting hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221, size=0 (0bytes) at 1731616245612Finishing writing output for hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 so closing down at 1731616249623 (+4011 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000003-wal.1731616245221.temp at 1731616249627 (+4 ms)3 split writer threads finished at 1731616249627Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000003-wal.1731616245221.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731616249635 (+8 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000003-wal.1731616245221.temp to hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000008 at 1731616249637 (+2 ms)Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221, size=0, length=0, corrupted=false, cancelled=false at 1731616249637 2024-11-14T20:30:49,638 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616245221 2024-11-14T20:30:49,639 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000008 2024-11-14T20:30:49,639 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:49,641 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:49,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616249641, exclude list is [], retry=0 2024-11-14T20:30:49,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:49,663 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:49,664 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:49,666 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616249641 2024-11-14T20:30:49,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:49,668 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b2a5ef733f596f2369a9662d21c49807, NAME => 'testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:49,669 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:49,669 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,669 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,670 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,671 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName a 2024-11-14T20:30:49,672 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,681 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/67b8f94eec974b48ac20c5563d6c91b6 2024-11-14T20:30:49,686 DEBUG [StoreFileOpener-b2a5ef733f596f2369a9662d21c49807-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:49,686 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_ 2024-11-14T20:30:49,690 DEBUG [StoreFileOpener-b2a5ef733f596f2369a9662d21c49807-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e079a4700fa44456a49d2324861777c7_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:49,691 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_ 2024-11-14T20:30:49,705 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b 2024-11-14T20:30:49,711 DEBUG [StoreFileOpener-b2a5ef733f596f2369a9662d21c49807-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for fd1b31f0db9a467382d1234cb67664c0_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-14T20:30:49,711 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_ 2024-11-14T20:30:49,711 WARN [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4176239e 2024-11-14T20:30:49,711 WARN [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4176239e 2024-11-14T20:30:49,711 WARN [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4176239e 2024-11-14T20:30:49,711 WARN [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@4176239e 2024-11-14T20:30:49,711 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_] to archive 2024-11-14T20:30:49,712 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-14T20:30:49,714 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_ to hdfs://localhost:43677/hbase/archive/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_ 2024-11-14T20:30:49,716 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_ to hdfs://localhost:43677/hbase/archive/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/e079a4700fa44456a49d2324861777c7_SeqId_4_ 2024-11-14T20:30:49,717 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b to hdfs://localhost:43677/hbase/archive/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/f6963d66ed304f89b0fdee8b726c133b 2024-11-14T20:30:49,719 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_ to hdfs://localhost:43677/hbase/archive/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/a/fd1b31f0db9a467382d1234cb67664c0_SeqId_4_ 2024-11-14T20:30:49,719 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,719 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,720 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName b 2024-11-14T20:30:49,720 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,721 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,721 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,721 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a5ef733f596f2369a9662d21c49807 columnFamilyName c 2024-11-14T20:30:49,721 DEBUG [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,722 INFO [StoreOpener-b2a5ef733f596f2369a9662d21c49807-1 {}] regionserver.HStore(327): Store=b2a5ef733f596f2369a9662d21c49807/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,722 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,723 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,724 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,725 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000008 2024-11-14T20:30:49,726 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:49,729 DEBUG [Time-limited test {}] regionserver.HRegion(5836): b2a5ef733f596f2369a9662d21c49807 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "b2a5ef733f596f2369a9662d21c49807" family_name: "a" compaction_input: "f6963d66ed304f89b0fdee8b726c133b" compaction_input: "c19e52ffbfed46138eba0d88d96aec3c_SeqId_4_" compaction_input: "e079a4700fa44456a49d2324861777c7_SeqId_4_" compaction_input: "fd1b31f0db9a467382d1234cb67664c0_SeqId_4_" compaction_output: "67b8f94eec974b48ac20c5563d6c91b6" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-14T20:30:49,729 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-14T20:30:49,729 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000008 2024-11-14T20:30:49,730 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/0000000000000000008 2024-11-14T20:30:49,731 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,731 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,732 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:49,733 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b2a5ef733f596f2369a9662d21c49807 2024-11-14T20:30:49,736 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testCompactedBulkLoadedFiles/b2a5ef733f596f2369a9662d21c49807/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-14T20:30:49,737 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b2a5ef733f596f2369a9662d21c49807; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68534114, jitterRate=0.021237879991531372}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:49,738 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b2a5ef733f596f2369a9662d21c49807: Writing region info on filesystem at 1731616249669Initializing all the Stores at 1731616249670 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249670Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249670Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249670Cleaning up temporary data from old regions at 1731616249731 (+61 ms)Region opened successfully at 1731616249737 (+6 ms) 2024-11-14T20:30:49,740 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b2a5ef733f596f2369a9662d21c49807, disabling compactions & flushes 2024-11-14T20:30:49,740 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:49,740 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:49,740 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. after waiting 0 ms 2024-11-14T20:30:49,740 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:49,742 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731616245038.b2a5ef733f596f2369a9662d21c49807. 2024-11-14T20:30:49,742 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b2a5ef733f596f2369a9662d21c49807: Waiting for close lock at 1731616249740Disabling compacts and flushes for region at 1731616249740Disabling writes for close at 1731616249740Writing region close event to WAL at 1731616249742 (+2 ms)Closed at 1731616249742 2024-11-14T20:30:49,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741929_1108 (size=95) 2024-11-14T20:30:49,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741929_1108 (size=95) 2024-11-14T20:30:49,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741929_1108 (size=95) 2024-11-14T20:30:49,748 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:49,748 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731616249641) 2024-11-14T20:30:49,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=432 (was 426) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43677 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-59053812_22 at /127.0.0.1:56268 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-59053812_22 at /127.0.0.1:49740 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43677 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-59053812_22 at /127.0.0.1:38656 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1342 (was 1266) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=721 (was 749), ProcessCount=11 (was 11), AvailableMemoryMB=5586 (was 5291) - AvailableMemoryMB LEAK? - 2024-11-14T20:30:49,771 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-14T20:30:49,791 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=432, OpenFileDescriptor=1342, MaxFileDescriptor=1048576, SystemLoadAverage=721, ProcessCount=11, AvailableMemoryMB=5583 2024-11-14T20:30:49,791 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1342 is superior to 1024 2024-11-14T20:30:49,810 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:49,811 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-14T20:30:49,812 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-14T20:30:49,815 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73463485, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/hregion-73463485, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:49,831 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73463485/hregion-73463485.1731616249816, exclude list is [], retry=0 2024-11-14T20:30:49,834 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:49,834 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:49,834 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:49,837 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73463485/hregion-73463485.1731616249816 2024-11-14T20:30:49,840 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:49,841 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b1e44adfaec3f982ec06051385e326b3, NAME => 'testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43677/hbase 2024-11-14T20:30:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741931_1110 (size=67) 2024-11-14T20:30:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741931_1110 (size=67) 2024-11-14T20:30:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741931_1110 (size=67) 2024-11-14T20:30:49,865 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:49,866 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,868 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName a 2024-11-14T20:30:49,868 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,869 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,870 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,871 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName b 2024-11-14T20:30:49,871 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,872 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,872 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,874 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName c 2024-11-14T20:30:49,874 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,874 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,875 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,875 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,876 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,877 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,877 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,878 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:49,879 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,882 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-14T20:30:49,882 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b1e44adfaec3f982ec06051385e326b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63146627, jitterRate=-0.059041932225227356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:49,882 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b1e44adfaec3f982ec06051385e326b3: Writing region info on filesystem at 1731616249865Initializing all the Stores at 1731616249866 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249866Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249866Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249866Cleaning up temporary data from old regions at 1731616249877 (+11 ms)Region opened successfully at 1731616249882 (+5 ms) 2024-11-14T20:30:49,883 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b1e44adfaec3f982ec06051385e326b3, disabling compactions & flushes 2024-11-14T20:30:49,883 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:49,883 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:49,883 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. after waiting 0 ms 2024-11-14T20:30:49,883 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:49,883 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:49,883 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b1e44adfaec3f982ec06051385e326b3: Waiting for close lock at 1731616249883Disabling compacts and flushes for region at 1731616249883Disabling writes for close at 1731616249883Writing region close event to WAL at 1731616249883Closed at 1731616249883 2024-11-14T20:30:49,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741930_1109 (size=95) 2024-11-14T20:30:49,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741930_1109 (size=95) 2024-11-14T20:30:49,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741930_1109 (size=95) 2024-11-14T20:30:49,890 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:49,890 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73463485:(num 1731616249816) 2024-11-14T20:30:49,890 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:49,892 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:49,913 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892, exclude list is [], retry=0 2024-11-14T20:30:49,915 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:49,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:49,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:49,918 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892 2024-11-14T20:30:49,919 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:49,919 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b1e44adfaec3f982ec06051385e326b3, NAME => 'testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:49,919 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:49,919 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,919 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,920 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,921 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName a 2024-11-14T20:30:49,921 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,922 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,922 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,923 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName b 2024-11-14T20:30:49,923 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,923 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,923 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,924 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName c 2024-11-14T20:30:49,924 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:49,925 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:49,925 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,927 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,927 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,928 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,928 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:49,929 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:49,931 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b1e44adfaec3f982ec06051385e326b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59220800, jitterRate=-0.11754131317138672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:49,931 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b1e44adfaec3f982ec06051385e326b3: Writing region info on filesystem at 1731616249919Initializing all the Stores at 1731616249920 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249920Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249920Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616249920Cleaning up temporary data from old regions at 1731616249928 (+8 ms)Region opened successfully at 1731616249931 (+3 ms) 2024-11-14T20:30:49,950 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b1e44adfaec3f982ec06051385e326b3 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-14T20:30:49,966 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/a/b58f4aa201204591823003790ebe49b8 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731616249931/Put/seqid=0 2024-11-14T20:30:49,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741933_1112 (size=5958) 2024-11-14T20:30:49,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741933_1112 (size=5958) 2024-11-14T20:30:49,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741933_1112 (size=5958) 2024-11-14T20:30:49,977 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/a/b58f4aa201204591823003790ebe49b8 2024-11-14T20:30:49,986 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/a/b58f4aa201204591823003790ebe49b8 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/a/b58f4aa201204591823003790ebe49b8 2024-11-14T20:30:49,991 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/a/b58f4aa201204591823003790ebe49b8, entries=10, sequenceid=13, filesize=5.8 K 2024-11-14T20:30:49,992 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for b1e44adfaec3f982ec06051385e326b3 in 43ms, sequenceid=13, compaction requested=false 2024-11-14T20:30:49,992 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b1e44adfaec3f982ec06051385e326b3: 2024-11-14T20:30:50,014 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b1e44adfaec3f982ec06051385e326b3, disabling compactions & flushes 2024-11-14T20:30:50,014 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:50,014 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:50,014 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. after waiting 0 ms 2024-11-14T20:30:50,014 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:50,015 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:50,015 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:50,015 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b1e44adfaec3f982ec06051385e326b3: Waiting for close lock at 1731616250014Disabling compacts and flushes for region at 1731616250014Disabling writes for close at 1731616250014Writing region close event to WAL at 1731616250015 (+1 ms)Closed at 1731616250015 2024-11-14T20:30:50,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741932_1111 (size=3345) 2024-11-14T20:30:50,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741932_1111 (size=3345) 2024-11-14T20:30:50,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741932_1111 (size=3345) 2024-11-14T20:30:50,041 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892, size=3.3 K (3345bytes) 2024-11-14T20:30:50,041 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892 2024-11-14T20:30:50,042 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892 after 1ms 2024-11-14T20:30:50,045 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:50,045 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892 took 4ms 2024-11-14T20:30:50,054 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892 so closing down 2024-11-14T20:30:50,054 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:50,055 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731616249892.temp 2024-11-14T20:30:50,057 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000003-wal.1731616249892.temp 2024-11-14T20:30:50,058 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:50,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741934_1113 (size=2944) 2024-11-14T20:30:50,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741934_1113 (size=2944) 2024-11-14T20:30:50,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741934_1113 (size=2944) 2024-11-14T20:30:50,105 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000003-wal.1731616249892.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:50,113 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000003-wal.1731616249892.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000035 2024-11-14T20:30:50,113 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 67 ms; skipped=2; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892, size=3.3 K, length=3345, corrupted=false, cancelled=false 2024-11-14T20:30:50,113 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892, journal: Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892, size=3.3 K (3345bytes) at 1731616250041Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892 so closing down at 1731616250054 (+13 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000003-wal.1731616249892.temp at 1731616250057 (+3 ms)3 split writer threads finished at 1731616250058 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000003-wal.1731616249892.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731616250105 (+47 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000003-wal.1731616249892.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000035 at 1731616250113 (+8 ms)Processed 32 edits across 1 Regions in 67 ms; skipped=2; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892, size=3.3 K, length=3345, corrupted=false, cancelled=false at 1731616250113 2024-11-14T20:30:50,117 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616249892 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616249892 2024-11-14T20:30:50,118 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000035 2024-11-14T20:30:50,119 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:50,130 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:50,154 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131, exclude list is [], retry=0 2024-11-14T20:30:50,158 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:50,158 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:50,158 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:50,173 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 2024-11-14T20:30:50,173 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43695:43695),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:42955:42955)] 2024-11-14T20:30:50,174 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b1e44adfaec3f982ec06051385e326b3, NAME => 'testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3.', STARTKEY => '', ENDKEY => ''} 2024-11-14T20:30:50,174 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:50,174 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,174 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,186 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,187 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName a 2024-11-14T20:30:50,188 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:50,208 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/a/b58f4aa201204591823003790ebe49b8 2024-11-14T20:30:50,208 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:50,208 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,209 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName b 2024-11-14T20:30:50,209 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:50,213 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:50,213 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,214 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName c 2024-11-14T20:30:50,214 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:50,214 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:50,215 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,217 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,218 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000035 2024-11-14T20:30:50,220 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:50,222 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000035 2024-11-14T20:30:50,222 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b1e44adfaec3f982ec06051385e326b3 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-14T20:30:50,245 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/b/0d67457f938e4eabab248ed5991457b7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731616249992/Put/seqid=0 2024-11-14T20:30:50,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741936_1115 (size=5958) 2024-11-14T20:30:50,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741936_1115 (size=5958) 2024-11-14T20:30:50,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741936_1115 (size=5958) 2024-11-14T20:30:50,285 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/b/0d67457f938e4eabab248ed5991457b7 2024-11-14T20:30:50,320 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/c/093ac30498e1461dbc9b549f6186de6d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731616250000/Put/seqid=0 2024-11-14T20:30:50,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741937_1116 (size=5958) 2024-11-14T20:30:50,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741937_1116 (size=5958) 2024-11-14T20:30:50,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741937_1116 (size=5958) 2024-11-14T20:30:50,344 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/c/093ac30498e1461dbc9b549f6186de6d 2024-11-14T20:30:50,352 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/b/0d67457f938e4eabab248ed5991457b7 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/b/0d67457f938e4eabab248ed5991457b7 2024-11-14T20:30:50,363 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/b/0d67457f938e4eabab248ed5991457b7, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T20:30:50,365 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/c/093ac30498e1461dbc9b549f6186de6d as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/c/093ac30498e1461dbc9b549f6186de6d 2024-11-14T20:30:50,375 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/c/093ac30498e1461dbc9b549f6186de6d, entries=10, sequenceid=35, filesize=5.8 K 2024-11-14T20:30:50,375 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for b1e44adfaec3f982ec06051385e326b3 in 153ms, sequenceid=35, compaction requested=false; wal=null 2024-11-14T20:30:50,376 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000035 2024-11-14T20:30:50,378 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,378 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,379 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:50,381 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:50,393 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-14T20:30:50,394 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b1e44adfaec3f982ec06051385e326b3; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62727895, jitterRate=-0.06528152525424957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:50,394 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b1e44adfaec3f982ec06051385e326b3: Writing region info on filesystem at 1731616250174Initializing all the Stores at 1731616250185 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616250185Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616250186 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616250186Obtaining lock to block concurrent updates at 1731616250222 (+36 ms)Preparing flush snapshotting stores in b1e44adfaec3f982ec06051385e326b3 at 1731616250222Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731616250222Flushing stores of testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. at 1731616250222Flushing b1e44adfaec3f982ec06051385e326b3/b: creating writer at 1731616250222Flushing b1e44adfaec3f982ec06051385e326b3/b: appending metadata at 1731616250244 (+22 ms)Flushing b1e44adfaec3f982ec06051385e326b3/b: closing flushed file at 1731616250244Flushing b1e44adfaec3f982ec06051385e326b3/c: creating writer at 1731616250292 (+48 ms)Flushing b1e44adfaec3f982ec06051385e326b3/c: appending metadata at 1731616250319 (+27 ms)Flushing b1e44adfaec3f982ec06051385e326b3/c: closing flushed file at 1731616250319Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77a4dbaa: reopening flushed file at 1731616250351 (+32 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61637271: reopening flushed file at 1731616250363 (+12 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for b1e44adfaec3f982ec06051385e326b3 in 153ms, sequenceid=35, compaction requested=false; wal=null at 1731616250375 (+12 ms)Cleaning up temporary data from old regions at 1731616250378 (+3 ms)Region opened successfully at 1731616250394 (+16 ms) 2024-11-14T20:30:50,484 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131, size=0 (0bytes) 2024-11-14T20:30:50,484 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 might be still open, length is 0 2024-11-14T20:30:50,484 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 2024-11-14T20:30:50,484 WARN [IPC Server handler 1 on default port 43677 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-14T20:30:50,485 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 after 0ms 2024-11-14T20:30:52,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:56352 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41951:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56352 dst: /127.0.0.1:41951 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41951 remote=/127.0.0.1:56352]. Total timeout mills is 60000, 57479 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:52,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:49786 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41701:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49786 dst: /127.0.0.1:41701 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:52,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1953420193_22 at /127.0.0.1:38720 [Receiving block BP-866344823-172.17.0.3-1731616199461:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37797:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38720 dst: /127.0.0.1:37797 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:52,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741935_1117 (size=2936) 2024-11-14T20:30:52,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741935_1117 (size=2936) 2024-11-14T20:30:53,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-14T20:30:53,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:53,835 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-14T20:30:53,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-14T20:30:54,485 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 after 4001ms 2024-11-14T20:30:54,489 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:54,489 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 took 4005ms 2024-11-14T20:30:54,494 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131; continuing. 2024-11-14T20:30:54,494 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 so closing down 2024-11-14T20:30:54,494 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-14T20:30:54,498 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731616250131.temp 2024-11-14T20:30:54,500 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000037-wal.1731616250131.temp 2024-11-14T20:30:54,500 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-14T20:30:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741938_1118 (size=2944) 2024-11-14T20:30:54,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741938_1118 (size=2944) 2024-11-14T20:30:54,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741938_1118 (size=2944) 2024-11-14T20:30:54,927 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000037-wal.1731616250131.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-14T20:30:54,929 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000037-wal.1731616250131.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000066 2024-11-14T20:30:54,929 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 437 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131, size=0, length=0, corrupted=false, cancelled=false 2024-11-14T20:30:54,929 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131, journal: Splitting hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131, size=0 (0bytes) at 1731616250484Finishing writing output for hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 so closing down at 1731616254494 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000037-wal.1731616250131.temp at 1731616254500 (+6 ms)3 split writer threads finished at 1731616254501 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000037-wal.1731616250131.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731616254927 (+426 ms)Rename recovered edits hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000037-wal.1731616250131.temp to hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000066 at 1731616254929 (+2 ms)Processed 30 edits across 1 Regions in 437 ms; skipped=0; WAL=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131, size=0, length=0, corrupted=false, cancelled=false at 1731616254929 2024-11-14T20:30:54,931 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 to hdfs://localhost:43677/hbase/oldWALs/wal.1731616250131 2024-11-14T20:30:54,932 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000066 2024-11-14T20:30:54,933 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-14T20:30:54,935 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43677/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809, archiveDir=hdfs://localhost:43677/hbase/oldWALs, maxLogs=32 2024-11-14T20:30:54,960 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616254935, exclude list is [], retry=0 2024-11-14T20:30:54,963 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37797,DS-e4cb8fa3-3aae-4d29-8fc9-9dc399fcb92c,DISK] 2024-11-14T20:30:54,964 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41701,DS-d8e7f6be-ab51-4af3-bd17-10a9b9ad2782,DISK] 2024-11-14T20:30:54,964 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41951,DS-829e2125-1cf6-4abe-92e7-10d5c245b269,DISK] 2024-11-14T20:30:55,002 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616254935 2024-11-14T20:30:55,005 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42955:42955),(127.0.0.1/127.0.0.1:39289:39289),(127.0.0.1/127.0.0.1:43695:43695)] 2024-11-14T20:30:55,006 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-14T20:30:55,013 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,017 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName a 2024-11-14T20:30:55,017 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:55,052 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/a/b58f4aa201204591823003790ebe49b8 2024-11-14T20:30:55,052 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:55,052 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,057 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName b 2024-11-14T20:30:55,057 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:55,104 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/b/0d67457f938e4eabab248ed5991457b7 2024-11-14T20:30:55,105 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:55,105 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,109 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1e44adfaec3f982ec06051385e326b3 columnFamilyName c 2024-11-14T20:30:55,109 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-14T20:30:55,125 DEBUG [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/c/093ac30498e1461dbc9b549f6186de6d 2024-11-14T20:30:55,125 INFO [StoreOpener-b1e44adfaec3f982ec06051385e326b3-1 {}] regionserver.HStore(327): Store=b1e44adfaec3f982ec06051385e326b3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-14T20:30:55,125 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,127 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,129 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,130 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000066 2024-11-14T20:30:55,138 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-11-14T20:30:55,145 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000066 2024-11-14T20:30:55,145 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b1e44adfaec3f982ec06051385e326b3 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-14T20:30:55,170 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/a/0058d988daaf45dda4ce409f03bd97c7 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731616250407/Put/seqid=0 2024-11-14T20:30:55,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741940_1120 (size=5958) 2024-11-14T20:30:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741940_1120 (size=5958) 2024-11-14T20:30:55,207 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/a/0058d988daaf45dda4ce409f03bd97c7 2024-11-14T20:30:55,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741940_1120 (size=5958) 2024-11-14T20:30:55,265 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/b/f4040ff25ba6436fb627489b97e534a4 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731616250423/Put/seqid=0 2024-11-14T20:30:55,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741941_1121 (size=5958) 2024-11-14T20:30:55,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741941_1121 (size=5958) 2024-11-14T20:30:55,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741941_1121 (size=5958) 2024-11-14T20:30:55,309 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/b/f4040ff25ba6436fb627489b97e534a4 2024-11-14T20:30:55,362 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/c/a02b3b7fb681419c9b9041f01336cfd9 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731616250430/Put/seqid=0 2024-11-14T20:30:55,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741942_1122 (size=5958) 2024-11-14T20:30:55,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741942_1122 (size=5958) 2024-11-14T20:30:55,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741942_1122 (size=5958) 2024-11-14T20:30:55,411 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/c/a02b3b7fb681419c9b9041f01336cfd9 2024-11-14T20:30:55,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/a/0058d988daaf45dda4ce409f03bd97c7 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/a/0058d988daaf45dda4ce409f03bd97c7 2024-11-14T20:30:55,482 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/a/0058d988daaf45dda4ce409f03bd97c7, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T20:30:55,504 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/b/f4040ff25ba6436fb627489b97e534a4 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/b/f4040ff25ba6436fb627489b97e534a4 2024-11-14T20:30:55,526 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/b/f4040ff25ba6436fb627489b97e534a4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T20:30:55,527 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/.tmp/c/a02b3b7fb681419c9b9041f01336cfd9 as hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/c/a02b3b7fb681419c9b9041f01336cfd9 2024-11-14T20:30:55,536 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/c/a02b3b7fb681419c9b9041f01336cfd9, entries=10, sequenceid=66, filesize=5.8 K 2024-11-14T20:30:55,536 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b1e44adfaec3f982ec06051385e326b3 in 391ms, sequenceid=66, compaction requested=false; wal=null 2024-11-14T20:30:55,537 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/0000000000000000066 2024-11-14T20:30:55,539 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,539 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,539 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-14T20:30:55,545 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b1e44adfaec3f982ec06051385e326b3 2024-11-14T20:30:55,550 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/hbase/data/default/testReplayEditsWrittenViaHRegion/b1e44adfaec3f982ec06051385e326b3/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-14T20:30:55,552 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b1e44adfaec3f982ec06051385e326b3; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61891040, jitterRate=-0.07775163650512695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-14T20:30:55,552 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b1e44adfaec3f982ec06051385e326b3: Writing region info on filesystem at 1731616255006Initializing all the Stores at 1731616255007 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616255007Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616255012 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731616255012Obtaining lock to block concurrent updates at 1731616255145 (+133 ms)Preparing flush snapshotting stores in b1e44adfaec3f982ec06051385e326b3 at 1731616255145Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731616255145Flushing stores of testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. at 1731616255145Flushing b1e44adfaec3f982ec06051385e326b3/a: creating writer at 1731616255146 (+1 ms)Flushing b1e44adfaec3f982ec06051385e326b3/a: appending metadata at 1731616255169 (+23 ms)Flushing b1e44adfaec3f982ec06051385e326b3/a: closing flushed file at 1731616255169Flushing b1e44adfaec3f982ec06051385e326b3/b: creating writer at 1731616255222 (+53 ms)Flushing b1e44adfaec3f982ec06051385e326b3/b: appending metadata at 1731616255256 (+34 ms)Flushing b1e44adfaec3f982ec06051385e326b3/b: closing flushed file at 1731616255257 (+1 ms)Flushing b1e44adfaec3f982ec06051385e326b3/c: creating writer at 1731616255325 (+68 ms)Flushing b1e44adfaec3f982ec06051385e326b3/c: appending metadata at 1731616255360 (+35 ms)Flushing b1e44adfaec3f982ec06051385e326b3/c: closing flushed file at 1731616255360Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59d7f96: reopening flushed file at 1731616255432 (+72 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a554f71: reopening flushed file at 1731616255482 (+50 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@351b79a1: reopening flushed file at 1731616255526 (+44 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for b1e44adfaec3f982ec06051385e326b3 in 391ms, sequenceid=66, compaction requested=false; wal=null at 1731616255536 (+10 ms)Cleaning up temporary data from old regions at 1731616255539 (+3 ms)Region opened successfully at 1731616255552 (+13 ms) 2024-11-14T20:30:55,584 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b1e44adfaec3f982ec06051385e326b3, disabling compactions & flushes 2024-11-14T20:30:55,584 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:55,584 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:55,584 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. after waiting 0 ms 2024-11-14T20:30:55,584 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:55,592 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731616249810.b1e44adfaec3f982ec06051385e326b3. 2024-11-14T20:30:55,592 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b1e44adfaec3f982ec06051385e326b3: Waiting for close lock at 1731616255584Disabling compacts and flushes for region at 1731616255584Disabling writes for close at 1731616255584Writing region close event to WAL at 1731616255592 (+8 ms)Closed at 1731616255592 2024-11-14T20:30:55,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741939_1119 (size=95) 2024-11-14T20:30:55,609 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616254935 not finished, retry = 0 2024-11-14T20:30:55,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741939_1119 (size=95) 2024-11-14T20:30:55,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741939_1119 (size=95) 2024-11-14T20:30:55,712 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-14T20:30:55,712 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731616254935) 2024-11-14T20:30:55,733 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=439 (was 432) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_329510279_22 at /127.0.0.1:38752 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_329510279_22 at /127.0.0.1:37942 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_329510279_22 at /127.0.0.1:37710 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43677 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1319272049) connection to localhost/127.0.0.1:43677 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1404 (was 1342) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=687 (was 721), ProcessCount=11 (was 11), AvailableMemoryMB=5219 (was 5583) 2024-11-14T20:30:55,733 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1404 is superior to 1024 2024-11-14T20:30:55,733 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-14T20:30:55,733 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T20:30:55,734 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T20:30:55,734 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:55,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:55,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-14T20:30:55,735 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-14T20:30:55,735 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=633453724, stopped=false 2024-11-14T20:30:55,735 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6c8768110af2,36945,1731616203247 2024-11-14T20:30:55,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T20:30:55,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:55,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T20:30:55,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-14T20:30:55,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:55,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:55,741 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T20:30:55,741 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T20:30:55,741 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-14T20:30:55,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T20:30:55,743 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-14T20:30:55,743 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T20:30:55,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:55,743 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6c8768110af2,46509,1731616204216' ***** 2024-11-14T20:30:55,743 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T20:30:55,743 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6c8768110af2,44657,1731616204351' ***** 2024-11-14T20:30:55,743 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-14T20:30:55,744 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T20:30:55,744 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-14T20:30:55,744 INFO [RS:1;6c8768110af2:44657 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T20:30:55,744 INFO [RS:0;6c8768110af2:46509 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-14T20:30:55,744 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T20:30:55,744 INFO [RS:1;6c8768110af2:44657 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T20:30:55,744 INFO [RS:0;6c8768110af2:46509 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-14T20:30:55,744 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(959): stopping server 6c8768110af2,44657,1731616204351 2024-11-14T20:30:55,744 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-14T20:30:55,744 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(3091): Received CLOSE for 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:55,744 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(959): stopping server 6c8768110af2,46509,1731616204216 2024-11-14T20:30:55,744 INFO [RS:0;6c8768110af2:46509 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T20:30:55,744 INFO [RS:0;6c8768110af2:46509 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6c8768110af2:46509. 2024-11-14T20:30:55,744 DEBUG [RS:0;6c8768110af2:46509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T20:30:55,744 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 18f7838e8e8a18bc6e359dc30256cd47, disabling compactions & flushes 2024-11-14T20:30:55,744 DEBUG [RS:0;6c8768110af2:46509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:55,744 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:55,745 INFO [RS:0;6c8768110af2:46509 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T20:30:55,745 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:55,745 INFO [RS:0;6c8768110af2:46509 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T20:30:55,745 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. after waiting 0 ms 2024-11-14T20:30:55,745 INFO [RS:0;6c8768110af2:46509 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T20:30:55,745 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:55,745 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-14T20:30:55,745 INFO [RS:1;6c8768110af2:44657 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T20:30:55,745 INFO [RS:1;6c8768110af2:44657 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6c8768110af2:44657. 2024-11-14T20:30:55,745 DEBUG [RS:1;6c8768110af2:44657 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-14T20:30:55,745 DEBUG [RS:1;6c8768110af2:44657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:55,745 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(976): stopping server 6c8768110af2,44657,1731616204351; all regions closed. 2024-11-14T20:30:55,748 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-14T20:30:55,748 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 18f7838e8e8a18bc6e359dc30256cd47=testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47.} 2024-11-14T20:30:55,748 DEBUG [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 18f7838e8e8a18bc6e359dc30256cd47 2024-11-14T20:30:55,748 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-14T20:30:55,748 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-14T20:30:55,748 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-14T20:30:55,748 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-14T20:30:55,748 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-14T20:30:55,748 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-14T20:30:55,761 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/info/f7b9385bcebb4a63bbad1068d50561b9 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47./info:regioninfo/1731616230694/Put/seqid=0 2024-11-14T20:30:55,770 INFO [regionserver/6c8768110af2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T20:30:55,776 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/default/testReplayEditsAfterRegionMovedWithMultiCF/18f7838e8e8a18bc6e359dc30256cd47/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-14T20:30:55,777 INFO [regionserver/6c8768110af2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T20:30:55,781 INFO [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:55,781 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 18f7838e8e8a18bc6e359dc30256cd47: Waiting for close lock at 1731616255744Running coprocessor pre-close hooks at 1731616255744Disabling compacts and flushes for region at 1731616255744Disabling writes for close at 1731616255745 (+1 ms)Writing region close event to WAL at 1731616255755 (+10 ms)Running coprocessor post-close hooks at 1731616255781 (+26 ms)Closed at 1731616255781 2024-11-14T20:30:55,781 DEBUG [RS_CLOSE_REGION-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731616225678.18f7838e8e8a18bc6e359dc30256cd47. 2024-11-14T20:30:55,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741835_1011 (size=95) 2024-11-14T20:30:55,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741835_1011 (size=95) 2024-11-14T20:30:55,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741835_1011 (size=95) 2024-11-14T20:30:55,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741943_1123 (size=6778) 2024-11-14T20:30:55,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741943_1123 (size=6778) 2024-11-14T20:30:55,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741943_1123 (size=6778) 2024-11-14T20:30:55,790 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/info/f7b9385bcebb4a63bbad1068d50561b9 2024-11-14T20:30:55,793 DEBUG [RS:1;6c8768110af2:44657 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs 2024-11-14T20:30:55,793 INFO [RS:1;6c8768110af2:44657 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6c8768110af2%2C44657%2C1731616204351:(num 1731616206886) 2024-11-14T20:30:55,793 DEBUG [RS:1;6c8768110af2:44657 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:55,793 INFO [RS:1;6c8768110af2:44657 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T20:30:55,793 INFO [RS:1;6c8768110af2:44657 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T20:30:55,793 INFO [RS:1;6c8768110af2:44657 {}] hbase.ChoreService(370): Chore service for: regionserver/6c8768110af2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-14T20:30:55,794 INFO [RS:1;6c8768110af2:44657 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-14T20:30:55,794 INFO [RS:1;6c8768110af2:44657 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-14T20:30:55,794 INFO [regionserver/6c8768110af2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T20:30:55,794 INFO [RS:1;6c8768110af2:44657 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-14T20:30:55,794 INFO [RS:1;6c8768110af2:44657 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T20:30:55,794 INFO [RS:1;6c8768110af2:44657 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44657 2024-11-14T20:30:55,797 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/.tmp/info/f7b9385bcebb4a63bbad1068d50561b9 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/info/f7b9385bcebb4a63bbad1068d50561b9 2024-11-14T20:30:55,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6c8768110af2,44657,1731616204351 2024-11-14T20:30:55,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T20:30:55,798 INFO [RS:1;6c8768110af2:44657 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T20:30:55,799 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6c8768110af2,44657,1731616204351] 2024-11-14T20:30:55,800 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6c8768110af2,44657,1731616204351 already deleted, retry=false 2024-11-14T20:30:55,801 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6c8768110af2,44657,1731616204351 expired; onlineServers=1 2024-11-14T20:30:55,803 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/info/f7b9385bcebb4a63bbad1068d50561b9, entries=8, sequenceid=23, filesize=6.6 K 2024-11-14T20:30:55,804 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=23, compaction requested=false 2024-11-14T20:30:55,804 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-14T20:30:55,829 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-14T20:30:55,830 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-14T20:30:55,830 INFO [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-14T20:30:55,830 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731616255748Running coprocessor pre-close hooks at 1731616255748Disabling compacts and flushes for region at 1731616255748Disabling writes for close at 1731616255748Obtaining lock to block concurrent updates at 1731616255748Preparing flush snapshotting stores in 1588230740 at 1731616255748Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731616255749 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731616255753 (+4 ms)Flushing 1588230740/info: creating writer at 1731616255753Flushing 1588230740/info: appending metadata at 1731616255760 (+7 ms)Flushing 1588230740/info: closing flushed file at 1731616255760Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10f733a1: reopening flushed file at 1731616255797 (+37 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 56ms, sequenceid=23, compaction requested=false at 1731616255804 (+7 ms)Writing region close event to WAL at 1731616255817 (+13 ms)Running coprocessor post-close hooks at 1731616255830 (+13 ms)Closed at 1731616255830 2024-11-14T20:30:55,830 DEBUG [RS_CLOSE_META-regionserver/6c8768110af2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-14T20:30:55,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:55,900 INFO [RS:1;6c8768110af2:44657 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T20:30:55,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44657-0x1003ffc70d80002, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:55,900 INFO [RS:1;6c8768110af2:44657 {}] regionserver.HRegionServer(1031): Exiting; stopping=6c8768110af2,44657,1731616204351; zookeeper connection closed. 2024-11-14T20:30:55,904 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@65d5b055 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@65d5b055 2024-11-14T20:30:55,948 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(976): stopping server 6c8768110af2,46509,1731616204216; all regions closed. 2024-11-14T20:30:55,963 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/WALs/6c8768110af2,46509,1731616204216/6c8768110af2%2C46509%2C1731616204216.meta.1731616229700.meta not finished, retry = 0 2024-11-14T20:30:55,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741893_1071 (size=1674) 2024-11-14T20:30:55,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741893_1071 (size=1674) 2024-11-14T20:30:55,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741893_1071 (size=1674) 2024-11-14T20:30:56,067 DEBUG [RS:0;6c8768110af2:46509 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs 2024-11-14T20:30:56,067 INFO [RS:0;6c8768110af2:46509 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6c8768110af2%2C46509%2C1731616204216.meta:.meta(num 1731616229700) 2024-11-14T20:30:56,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741834_1010 (size=2171) 2024-11-14T20:30:56,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741834_1010 (size=2171) 2024-11-14T20:30:56,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741834_1010 (size=2171) 2024-11-14T20:30:56,081 DEBUG [RS:0;6c8768110af2:46509 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/oldWALs 2024-11-14T20:30:56,081 INFO [RS:0;6c8768110af2:46509 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6c8768110af2%2C46509%2C1731616204216:(num 1731616206883) 2024-11-14T20:30:56,081 DEBUG [RS:0;6c8768110af2:46509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-14T20:30:56,081 INFO [RS:0;6c8768110af2:46509 {}] regionserver.LeaseManager(133): Closed leases 2024-11-14T20:30:56,081 INFO [RS:0;6c8768110af2:46509 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T20:30:56,081 INFO [RS:0;6c8768110af2:46509 {}] hbase.ChoreService(370): Chore service for: regionserver/6c8768110af2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-14T20:30:56,081 INFO [RS:0;6c8768110af2:46509 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T20:30:56,081 INFO [regionserver/6c8768110af2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T20:30:56,082 INFO [RS:0;6c8768110af2:46509 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46509 2024-11-14T20:30:56,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-14T20:30:56,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6c8768110af2,46509,1731616204216 2024-11-14T20:30:56,085 INFO [RS:0;6c8768110af2:46509 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T20:30:56,086 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6c8768110af2,46509,1731616204216] 2024-11-14T20:30:56,088 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6c8768110af2,46509,1731616204216 already deleted, retry=false 2024-11-14T20:30:56,088 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6c8768110af2,46509,1731616204216 expired; onlineServers=0 2024-11-14T20:30:56,089 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6c8768110af2,36945,1731616203247' ***** 2024-11-14T20:30:56,089 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-14T20:30:56,089 INFO [M:0;6c8768110af2:36945 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-14T20:30:56,089 INFO [M:0;6c8768110af2:36945 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-14T20:30:56,089 DEBUG [M:0;6c8768110af2:36945 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-14T20:30:56,089 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-14T20:30:56,089 DEBUG [M:0;6c8768110af2:36945 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-14T20:30:56,089 DEBUG [master/6c8768110af2:0:becomeActiveMaster-HFileCleaner.small.0-1731616206445 {}] cleaner.HFileCleaner(306): Exit Thread[master/6c8768110af2:0:becomeActiveMaster-HFileCleaner.small.0-1731616206445,5,FailOnTimeoutGroup] 2024-11-14T20:30:56,089 DEBUG [master/6c8768110af2:0:becomeActiveMaster-HFileCleaner.large.0-1731616206441 {}] cleaner.HFileCleaner(306): Exit Thread[master/6c8768110af2:0:becomeActiveMaster-HFileCleaner.large.0-1731616206441,5,FailOnTimeoutGroup] 2024-11-14T20:30:56,089 INFO [M:0;6c8768110af2:36945 {}] hbase.ChoreService(370): Chore service for: master/6c8768110af2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-14T20:30:56,090 INFO [M:0;6c8768110af2:36945 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-14T20:30:56,090 DEBUG [M:0;6c8768110af2:36945 {}] master.HMaster(1795): Stopping service threads 2024-11-14T20:30:56,090 INFO [M:0;6c8768110af2:36945 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-14T20:30:56,090 INFO [M:0;6c8768110af2:36945 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-14T20:30:56,090 INFO [M:0;6c8768110af2:36945 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-14T20:30:56,091 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-14T20:30:56,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-14T20:30:56,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-14T20:30:56,092 DEBUG [M:0;6c8768110af2:36945 {}] zookeeper.ZKUtil(347): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-14T20:30:56,092 WARN [M:0;6c8768110af2:36945 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-14T20:30:56,093 INFO [M:0;6c8768110af2:36945 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/.lastflushedseqids 2024-11-14T20:30:56,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741944_1124 (size=119) 2024-11-14T20:30:56,150 INFO [M:0;6c8768110af2:36945 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-14T20:30:56,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741944_1124 (size=119) 2024-11-14T20:30:56,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741944_1124 (size=119) 2024-11-14T20:30:56,151 INFO [M:0;6c8768110af2:36945 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-14T20:30:56,151 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-14T20:30:56,151 INFO [M:0;6c8768110af2:36945 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:56,151 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:56,151 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-14T20:30:56,151 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:56,151 INFO [M:0;6c8768110af2:36945 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-14T20:30:56,172 DEBUG [M:0;6c8768110af2:36945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9cd78d1126e744a5aa05c0cadf269957 is 82, key is hbase:meta,,1/info:regioninfo/1731616229968/Put/seqid=0 2024-11-14T20:30:56,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:56,188 INFO [RS:0;6c8768110af2:46509 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T20:30:56,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46509-0x1003ffc70d80001, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:56,188 INFO [RS:0;6c8768110af2:46509 {}] regionserver.HRegionServer(1031): Exiting; stopping=6c8768110af2,46509,1731616204216; zookeeper connection closed. 2024-11-14T20:30:56,198 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6dc42200 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6dc42200 2024-11-14T20:30:56,199 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-14T20:30:56,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741945_1125 (size=6063) 2024-11-14T20:30:56,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741945_1125 (size=6063) 2024-11-14T20:30:56,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741945_1125 (size=6063) 2024-11-14T20:30:56,221 INFO [M:0;6c8768110af2:36945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9cd78d1126e744a5aa05c0cadf269957 2024-11-14T20:30:56,258 DEBUG [M:0;6c8768110af2:36945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0920acbc5b0d4130a67befab4d7f8afb is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731616226234/Put/seqid=0 2024-11-14T20:30:56,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741946_1126 (size=7907) 2024-11-14T20:30:56,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741946_1126 (size=7907) 2024-11-14T20:30:56,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741946_1126 (size=7907) 2024-11-14T20:30:56,677 INFO [M:0;6c8768110af2:36945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0920acbc5b0d4130a67befab4d7f8afb 2024-11-14T20:30:56,682 INFO [M:0;6c8768110af2:36945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0920acbc5b0d4130a67befab4d7f8afb 2024-11-14T20:30:56,700 DEBUG [M:0;6c8768110af2:36945 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0ec74d3bdd14aeaa5af7f97e08e379d is 69, key is 6c8768110af2,44657,1731616204351/rs:state/1731616206554/Put/seqid=0 2024-11-14T20:30:56,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741947_1127 (size=5440) 2024-11-14T20:30:56,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741947_1127 (size=5440) 2024-11-14T20:30:56,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741947_1127 (size=5440) 2024-11-14T20:30:57,131 INFO [M:0;6c8768110af2:36945 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0ec74d3bdd14aeaa5af7f97e08e379d 2024-11-14T20:30:57,138 INFO [M:0;6c8768110af2:36945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0ec74d3bdd14aeaa5af7f97e08e379d 2024-11-14T20:30:57,139 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9cd78d1126e744a5aa05c0cadf269957 as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9cd78d1126e744a5aa05c0cadf269957 2024-11-14T20:30:57,145 INFO [M:0;6c8768110af2:36945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9cd78d1126e744a5aa05c0cadf269957, entries=14, sequenceid=207, filesize=5.9 K 2024-11-14T20:30:57,146 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0920acbc5b0d4130a67befab4d7f8afb as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0920acbc5b0d4130a67befab4d7f8afb 2024-11-14T20:30:57,152 INFO [M:0;6c8768110af2:36945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0920acbc5b0d4130a67befab4d7f8afb 2024-11-14T20:30:57,152 INFO [M:0;6c8768110af2:36945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0920acbc5b0d4130a67befab4d7f8afb, entries=21, sequenceid=207, filesize=7.7 K 2024-11-14T20:30:57,157 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e0ec74d3bdd14aeaa5af7f97e08e379d as hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0ec74d3bdd14aeaa5af7f97e08e379d 2024-11-14T20:30:57,166 INFO [M:0;6c8768110af2:36945 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e0ec74d3bdd14aeaa5af7f97e08e379d 2024-11-14T20:30:57,166 INFO [M:0;6c8768110af2:36945 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43677/user/jenkins/test-data/a695fc2a-49d9-3b11-9b98-249102efa668/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e0ec74d3bdd14aeaa5af7f97e08e379d, entries=3, sequenceid=207, filesize=5.3 K 2024-11-14T20:30:57,167 INFO [M:0;6c8768110af2:36945 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1016ms, sequenceid=207, compaction requested=false 2024-11-14T20:30:57,188 INFO [M:0;6c8768110af2:36945 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-14T20:30:57,188 DEBUG [M:0;6c8768110af2:36945 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731616256151Disabling compacts and flushes for region at 1731616256151Disabling writes for close at 1731616256151Obtaining lock to block concurrent updates at 1731616256151Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731616256151Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1731616256152 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731616256153 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731616256153Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731616256172 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731616256172Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731616256227 (+55 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731616256257 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731616256257Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731616256682 (+425 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731616256698 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731616256698Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ba39e9a: reopening flushed file at 1731616257138 (+440 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@269c613: reopening flushed file at 1731616257145 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cdbb187: reopening flushed file at 1731616257152 (+7 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1016ms, sequenceid=207, compaction requested=false at 1731616257167 (+15 ms)Writing region close event to WAL at 1731616257188 (+21 ms)Closed at 1731616257188 2024-11-14T20:30:57,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37797 is added to blk_1073741830_1006 (size=69522) 2024-11-14T20:30:57,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41701 is added to blk_1073741830_1006 (size=69522) 2024-11-14T20:30:57,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41951 is added to blk_1073741830_1006 (size=69522) 2024-11-14T20:30:57,205 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-14T20:30:57,205 INFO [M:0;6c8768110af2:36945 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-14T20:30:57,205 INFO [M:0;6c8768110af2:36945 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36945 2024-11-14T20:30:57,205 INFO [M:0;6c8768110af2:36945 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-14T20:30:57,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:57,309 INFO [M:0;6c8768110af2:36945 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-14T20:30:57,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36945-0x1003ffc70d80000, quorum=127.0.0.1:60776, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-14T20:30:57,318 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616242295 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616242295 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731616232789/wal.1731616242295 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T20:30:57,323 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616232616 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616232616 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731616231249/wal.1731616232616 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T20:30:57,324 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731616220959/wal.1731616221139 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:57,327 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616244519 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616244519 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731616242676/wal.1731616244519 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T20:30:57,328 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731616245037/wal.1731616245221 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:57,328 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731616249809/wal.1731616250131 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:57,329 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731616211773/wal.1731616212141 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-14T20:30:57,332 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal.1731616211360 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal.1731616211360 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731616210547/wal.1731616211360 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T20:30:57,335 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal.1731616209206 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal.1731616209206 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731616208057/wal.1731616209206 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1953420193_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-14T20:30:57,367 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@35f1150e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T20:30:57,375 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13a77e13{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T20:30:57,375 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T20:30:57,375 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f9972d0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T20:30:57,375 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a6d5e13{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,STOPPED} 2024-11-14T20:30:57,393 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T20:30:57,393 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T20:30:57,396 WARN [BP-866344823-172.17.0.3-1731616199461 heartbeating to localhost/127.0.0.1:43677 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T20:30:57,397 WARN [BP-866344823-172.17.0.3-1731616199461 heartbeating to localhost/127.0.0.1:43677 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-866344823-172.17.0.3-1731616199461 (Datanode Uuid bfdfaea9-1ab9-4521-ba18-c32481d43c60) service to localhost/127.0.0.1:43677 2024-11-14T20:30:57,398 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data5/current/BP-866344823-172.17.0.3-1731616199461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T20:30:57,399 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T20:30:57,400 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data6/current/BP-866344823-172.17.0.3-1731616199461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T20:30:57,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7bd427b8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T20:30:57,409 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6915083f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T20:30:57,409 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T20:30:57,409 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cc2d6b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T20:30:57,409 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46b092e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,STOPPED} 2024-11-14T20:30:57,417 WARN [BP-866344823-172.17.0.3-1731616199461 heartbeating to localhost/127.0.0.1:43677 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T20:30:57,417 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T20:30:57,417 WARN [BP-866344823-172.17.0.3-1731616199461 heartbeating to localhost/127.0.0.1:43677 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-866344823-172.17.0.3-1731616199461 (Datanode Uuid 1428fba7-9712-4cef-8ee1-65176d395f14) service to localhost/127.0.0.1:43677 2024-11-14T20:30:57,417 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T20:30:57,418 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T20:30:57,418 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data4/current/BP-866344823-172.17.0.3-1731616199461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T20:30:57,420 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data3/current/BP-866344823-172.17.0.3-1731616199461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T20:30:57,429 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@330740de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-14T20:30:57,430 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7b24cab9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T20:30:57,430 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T20:30:57,430 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a359997{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T20:30:57,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@cf5a85e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,STOPPED} 2024-11-14T20:30:57,442 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-14T20:30:57,442 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-14T20:30:57,444 WARN [BP-866344823-172.17.0.3-1731616199461 heartbeating to localhost/127.0.0.1:43677 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-14T20:30:57,444 WARN [BP-866344823-172.17.0.3-1731616199461 heartbeating to localhost/127.0.0.1:43677 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-866344823-172.17.0.3-1731616199461 (Datanode Uuid dd75bad4-f7b2-4f9a-9203-8adb2036a1e3) service to localhost/127.0.0.1:43677 2024-11-14T20:30:57,446 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data2/current/BP-866344823-172.17.0.3-1731616199461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T20:30:57,447 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-14T20:30:57,448 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/cluster_f87179a4-a128-1cfc-7eb8-d959ceee4eb4/data/data1/current/BP-866344823-172.17.0.3-1731616199461 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-14T20:30:57,471 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3717288f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-14T20:30:57,476 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4bd70930{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-14T20:30:57,476 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-14T20:30:57,476 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dc9d5c1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-14T20:30:57,476 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4f37ffca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/98cd9a7b-dc51-c018-e6cb-e5cc05a87c38/hadoop.log.dir/,STOPPED} 2024-11-14T20:30:57,507 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-14T20:30:57,589 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down