2024-11-13 19:53:49,637 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 19:53:49,653 main DEBUG Took 0.012972 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 19:53:49,653 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 19:53:49,654 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 19:53:49,655 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 19:53:49,656 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,665 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 19:53:49,681 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,683 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,684 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,684 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,685 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,685 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,687 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,687 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,688 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,688 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,689 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,689 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,690 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,690 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,691 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,691 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,692 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,693 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,693 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,694 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,695 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,695 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:53:49,696 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,696 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 19:53:49,699 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:53:49,700 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 19:53:49,702 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 19:53:49,703 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 19:53:49,704 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 19:53:49,705 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 19:53:49,720 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 19:53:49,723 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 19:53:49,725 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 19:53:49,725 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 19:53:49,726 main DEBUG createAppenders(={Console}) 2024-11-13 19:53:49,727 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-13 19:53:49,727 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-13 19:53:49,728 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-13 19:53:49,728 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 19:53:49,729 main DEBUG OutputStream closed 2024-11-13 19:53:49,729 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 19:53:49,729 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 19:53:49,730 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-13 19:53:49,818 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 19:53:49,821 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 19:53:49,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 19:53:49,822 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 19:53:49,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 19:53:49,823 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 19:53:49,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 19:53:49,824 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 19:53:49,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 19:53:49,825 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 19:53:49,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 19:53:49,826 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 19:53:49,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 19:53:49,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 19:53:49,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 19:53:49,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 19:53:49,828 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 19:53:49,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 19:53:49,831 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 19:53:49,832 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-13 19:53:49,832 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 19:53:49,833 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-13T19:53:50,201 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1 2024-11-13 19:53:50,203 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 19:53:50,204 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T19:53:50,213 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T19:53:50,219 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-13T19:53:50,241 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T19:53:50,289 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T19:53:50,290 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T19:53:50,302 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T19:53:50,322 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4, deleteOnExit=true 2024-11-13T19:53:50,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T19:53:50,323 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/test.cache.data in system properties and HBase conf 2024-11-13T19:53:50,324 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T19:53:50,325 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir in system properties and HBase conf 2024-11-13T19:53:50,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T19:53:50,326 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T19:53:50,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T19:53:50,424 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T19:53:50,549 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T19:53:50,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T19:53:50,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T19:53:50,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T19:53:50,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T19:53:50,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T19:53:50,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T19:53:50,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T19:53:50,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T19:53:50,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T19:53:50,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/nfs.dump.dir in system properties and HBase conf 2024-11-13T19:53:50,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/java.io.tmpdir in system properties and HBase conf 2024-11-13T19:53:50,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T19:53:50,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T19:53:50,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T19:53:51,473 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T19:53:51,549 INFO [Time-limited test {}] log.Log(170): Logging initialized @2871ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T19:53:51,629 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:53:51,713 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:53:51,751 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:53:51,752 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:53:51,753 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T19:53:51,777 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:53:51,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:53:51,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:53:52,013 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5608f426{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/java.io.tmpdir/jetty-localhost-44077-hadoop-hdfs-3_4_1-tests_jar-_-any-15647610807432829865/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T19:53:52,021 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:44077} 2024-11-13T19:53:52,022 INFO [Time-limited test {}] server.Server(415): Started @3344ms 2024-11-13T19:53:52,445 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:53:52,454 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:53:52,456 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:53:52,456 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:53:52,457 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T19:53:52,457 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fcc4f36{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:53:52,458 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cf706e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:53:52,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1be71666{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/java.io.tmpdir/jetty-localhost-38167-hadoop-hdfs-3_4_1-tests_jar-_-any-12682196023795669929/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:53:52,590 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@9edd34e{HTTP/1.1, (http/1.1)}{localhost:38167} 2024-11-13T19:53:52,591 INFO [Time-limited test {}] server.Server(415): Started @3913ms 2024-11-13T19:53:52,646 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T19:53:52,786 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:53:52,795 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:53:52,800 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:53:52,800 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:53:52,800 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T19:53:52,801 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f9065a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:53:52,802 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b034666{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:53:52,972 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5705c253{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/java.io.tmpdir/jetty-localhost-38611-hadoop-hdfs-3_4_1-tests_jar-_-any-17538948559038557280/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:53:52,973 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5af9d354{HTTP/1.1, (http/1.1)}{localhost:38611} 2024-11-13T19:53:52,974 INFO [Time-limited test {}] server.Server(415): Started @4296ms 2024-11-13T19:53:52,976 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T19:53:53,064 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:53:53,071 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:53:53,079 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:53:53,080 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:53:53,080 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T19:53:53,082 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16c07ee3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:53:53,083 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7161498b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:53:53,205 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data3/current/BP-1458969915-172.17.0.2-1731527631214/current, will proceed with Du for space computation calculation, 2024-11-13T19:53:53,208 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data1/current/BP-1458969915-172.17.0.2-1731527631214/current, will proceed with Du for space computation calculation, 2024-11-13T19:53:53,210 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data2/current/BP-1458969915-172.17.0.2-1731527631214/current, will proceed with Du for space computation calculation, 2024-11-13T19:53:53,213 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data4/current/BP-1458969915-172.17.0.2-1731527631214/current, will proceed with Du for space computation calculation, 2024-11-13T19:53:53,233 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5deffb5d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/java.io.tmpdir/jetty-localhost-41969-hadoop-hdfs-3_4_1-tests_jar-_-any-4104985255602705070/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:53:53,234 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5a646eb4{HTTP/1.1, (http/1.1)}{localhost:41969} 2024-11-13T19:53:53,234 INFO [Time-limited test {}] server.Server(415): Started @4557ms 2024-11-13T19:53:53,237 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T19:53:53,286 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T19:53:53,287 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T19:53:53,382 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d978fa390488c8f with lease ID 0xb4c5ef563c0af656: Processing first storage report for DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5 from datanode DatanodeRegistration(127.0.0.1:43261, datanodeUuid=6b11cc4b-8d1d-403a-a10f-829b720687bb, infoPort=42917, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214) 2024-11-13T19:53:53,384 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d978fa390488c8f with lease ID 0xb4c5ef563c0af656: from storage DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5 node DatanodeRegistration(127.0.0.1:43261, datanodeUuid=6b11cc4b-8d1d-403a-a10f-829b720687bb, infoPort=42917, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T19:53:53,384 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3b981864579f71d with lease ID 0xb4c5ef563c0af655: Processing first storage report for DS-7859c527-ec0a-41b8-80ba-8646ff04a519 from datanode DatanodeRegistration(127.0.0.1:33527, datanodeUuid=3b4f5e91-0f84-4e4e-b294-53a707cf967f, infoPort=46605, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214) 2024-11-13T19:53:53,385 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3b981864579f71d with lease ID 0xb4c5ef563c0af655: from storage DS-7859c527-ec0a-41b8-80ba-8646ff04a519 node DatanodeRegistration(127.0.0.1:33527, datanodeUuid=3b4f5e91-0f84-4e4e-b294-53a707cf967f, infoPort=46605, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T19:53:53,385 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9d978fa390488c8f with lease ID 0xb4c5ef563c0af656: Processing first storage report for DS-065de6f1-1dbd-4472-af1f-692d99a1e294 from datanode DatanodeRegistration(127.0.0.1:43261, datanodeUuid=6b11cc4b-8d1d-403a-a10f-829b720687bb, infoPort=42917, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214) 2024-11-13T19:53:53,385 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9d978fa390488c8f with lease ID 0xb4c5ef563c0af656: from storage DS-065de6f1-1dbd-4472-af1f-692d99a1e294 node DatanodeRegistration(127.0.0.1:43261, datanodeUuid=6b11cc4b-8d1d-403a-a10f-829b720687bb, infoPort=42917, infoSecurePort=0, ipcPort=46281, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T19:53:53,385 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd3b981864579f71d with lease ID 0xb4c5ef563c0af655: Processing first storage report for DS-67d607d3-fd39-4b81-b43f-bd4e483951f3 from datanode DatanodeRegistration(127.0.0.1:33527, datanodeUuid=3b4f5e91-0f84-4e4e-b294-53a707cf967f, infoPort=46605, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214) 2024-11-13T19:53:53,386 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd3b981864579f71d with lease ID 0xb4c5ef563c0af655: from storage DS-67d607d3-fd39-4b81-b43f-bd4e483951f3 node DatanodeRegistration(127.0.0.1:33527, datanodeUuid=3b4f5e91-0f84-4e4e-b294-53a707cf967f, infoPort=46605, infoSecurePort=0, ipcPort=32851, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T19:53:53,426 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data5/current/BP-1458969915-172.17.0.2-1731527631214/current, will proceed with Du for space computation calculation, 2024-11-13T19:53:53,436 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data6/current/BP-1458969915-172.17.0.2-1731527631214/current, will proceed with Du for space computation calculation, 2024-11-13T19:53:53,480 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T19:53:53,486 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84f8892f82ed4d42 with lease ID 0xb4c5ef563c0af657: Processing first storage report for DS-c00217f9-8b82-40cf-9fec-6506b10c0441 from datanode DatanodeRegistration(127.0.0.1:33541, datanodeUuid=962dc963-8913-476b-9d54-664e01726004, infoPort=35753, infoSecurePort=0, ipcPort=39169, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214) 2024-11-13T19:53:53,487 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84f8892f82ed4d42 with lease ID 0xb4c5ef563c0af657: from storage DS-c00217f9-8b82-40cf-9fec-6506b10c0441 node DatanodeRegistration(127.0.0.1:33541, datanodeUuid=962dc963-8913-476b-9d54-664e01726004, infoPort=35753, infoSecurePort=0, ipcPort=39169, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T19:53:53,487 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x84f8892f82ed4d42 with lease ID 0xb4c5ef563c0af657: Processing first storage report for DS-a6afb80b-ce36-470a-9f9c-24109ee9483c from datanode DatanodeRegistration(127.0.0.1:33541, datanodeUuid=962dc963-8913-476b-9d54-664e01726004, infoPort=35753, infoSecurePort=0, ipcPort=39169, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214) 2024-11-13T19:53:53,487 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x84f8892f82ed4d42 with lease ID 0xb4c5ef563c0af657: from storage DS-a6afb80b-ce36-470a-9f9c-24109ee9483c node DatanodeRegistration(127.0.0.1:33541, datanodeUuid=962dc963-8913-476b-9d54-664e01726004, infoPort=35753, infoSecurePort=0, ipcPort=39169, storageInfo=lv=-57;cid=testClusterID;nsid=199501588;c=1731527631214), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T19:53:53,709 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1 2024-11-13T19:53:53,799 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/zookeeper_0, clientPort=57594, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T19:53:53,808 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57594 2024-11-13T19:53:53,834 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:53,836 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:54,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741825_1001 (size=7) 2024-11-13T19:53:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741825_1001 (size=7) 2024-11-13T19:53:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741825_1001 (size=7) 2024-11-13T19:53:54,528 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976 with version=8 2024-11-13T19:53:54,528 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/hbase-staging 2024-11-13T19:53:54,865 INFO [Time-limited test {}] client.ConnectionUtils(128): master/54d0784d1943:0 server-side Connection retries=45 2024-11-13T19:53:54,875 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:54,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:54,881 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:53:54,881 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:54,881 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:53:55,063 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T19:53:55,136 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T19:53:55,147 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T19:53:55,152 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:53:55,191 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4144 (auto-detected) 2024-11-13T19:53:55,192 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T19:53:55,221 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35115 2024-11-13T19:53:55,252 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35115 connecting to ZooKeeper ensemble=127.0.0.1:57594 2024-11-13T19:53:55,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:351150x0, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:53:55,308 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35115-0x100eda9e0200000 connected 2024-11-13T19:53:55,355 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,360 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,376 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:53:55,381 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976, hbase.cluster.distributed=false 2024-11-13T19:53:55,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:53:55,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35115 2024-11-13T19:53:55,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35115 2024-11-13T19:53:55,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35115 2024-11-13T19:53:55,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35115 2024-11-13T19:53:55,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35115 2024-11-13T19:53:55,585 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/54d0784d1943:0 server-side Connection retries=45 2024-11-13T19:53:55,587 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,588 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:53:55,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,588 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:53:55,591 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T19:53:55,594 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:53:55,595 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46741 2024-11-13T19:53:55,598 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46741 connecting to ZooKeeper ensemble=127.0.0.1:57594 2024-11-13T19:53:55,599 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,604 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467410x0, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:53:55,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:53:55,615 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46741-0x100eda9e0200001 connected 2024-11-13T19:53:55,622 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T19:53:55,636 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T19:53:55,639 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T19:53:55,645 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:53:55,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46741 2024-11-13T19:53:55,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46741 2024-11-13T19:53:55,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46741 2024-11-13T19:53:55,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46741 2024-11-13T19:53:55,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46741 2024-11-13T19:53:55,668 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/54d0784d1943:0 server-side Connection retries=45 2024-11-13T19:53:55,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,669 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:53:55,669 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,670 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:53:55,670 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T19:53:55,670 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:53:55,671 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36813 2024-11-13T19:53:55,673 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36813 connecting to ZooKeeper ensemble=127.0.0.1:57594 2024-11-13T19:53:55,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,676 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368130x0, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:53:55,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36813-0x100eda9e0200002 connected 2024-11-13T19:53:55,684 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:53:55,685 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T19:53:55,687 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T19:53:55,688 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T19:53:55,690 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:53:55,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36813 2024-11-13T19:53:55,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36813 2024-11-13T19:53:55,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36813 2024-11-13T19:53:55,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36813 2024-11-13T19:53:55,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36813 2024-11-13T19:53:55,713 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/54d0784d1943:0 server-side Connection retries=45 2024-11-13T19:53:55,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,713 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:53:55,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:53:55,713 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:53:55,713 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T19:53:55,714 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:53:55,715 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35943 2024-11-13T19:53:55,716 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35943 connecting to ZooKeeper ensemble=127.0.0.1:57594 2024-11-13T19:53:55,717 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,719 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359430x0, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:53:55,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359430x0, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:53:55,725 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35943-0x100eda9e0200003 connected 2024-11-13T19:53:55,726 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T19:53:55,727 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T19:53:55,728 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T19:53:55,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:53:55,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35943 2024-11-13T19:53:55,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35943 2024-11-13T19:53:55,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35943 2024-11-13T19:53:55,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35943 2024-11-13T19:53:55,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35943 2024-11-13T19:53:55,752 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;54d0784d1943:35115 2024-11-13T19:53:55,753 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/54d0784d1943,35115,1731527634648 2024-11-13T19:53:55,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,764 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/54d0784d1943,35115,1731527634648 2024-11-13T19:53:55,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T19:53:55,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T19:53:55,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T19:53:55,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,786 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T19:53:55,787 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/54d0784d1943,35115,1731527634648 from backup master directory 2024-11-13T19:53:55,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/54d0784d1943,35115,1731527634648 2024-11-13T19:53:55,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:53:55,792 WARN [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:53:55,792 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=54d0784d1943,35115,1731527634648 2024-11-13T19:53:55,794 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T19:53:55,795 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T19:53:55,858 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/hbase.id] with ID: 697cb3e1-bdea-4df8-92f0-0da5980bf2ec 2024-11-13T19:53:55,859 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/.tmp/hbase.id 2024-11-13T19:53:55,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741826_1002 (size=42) 2024-11-13T19:53:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741826_1002 (size=42) 2024-11-13T19:53:55,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741826_1002 (size=42) 2024-11-13T19:53:55,876 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/.tmp/hbase.id]:[hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/hbase.id] 2024-11-13T19:53:55,924 INFO [master/54d0784d1943:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:53:55,929 INFO [master/54d0784d1943:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T19:53:55,948 INFO [master/54d0784d1943:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-13T19:53:55,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:55,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741827_1003 (size=196) 2024-11-13T19:53:55,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741827_1003 (size=196) 2024-11-13T19:53:55,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741827_1003 (size=196) 2024-11-13T19:53:55,996 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:53:55,998 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T19:53:56,005 INFO [master/54d0784d1943:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:53:56,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741828_1004 (size=1189) 2024-11-13T19:53:56,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741828_1004 (size=1189) 2024-11-13T19:53:56,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741828_1004 (size=1189) 2024-11-13T19:53:56,065 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store 2024-11-13T19:53:56,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741829_1005 (size=34) 2024-11-13T19:53:56,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741829_1005 (size=34) 2024-11-13T19:53:56,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741829_1005 (size=34) 2024-11-13T19:53:56,104 INFO [master/54d0784d1943:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T19:53:56,108 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:53:56,110 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T19:53:56,110 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:53:56,110 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:53:56,112 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T19:53:56,112 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:53:56,113 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:53:56,114 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731527636110Disabling compacts and flushes for region at 1731527636110Disabling writes for close at 1731527636112 (+2 ms)Writing region close event to WAL at 1731527636113 (+1 ms)Closed at 1731527636113 2024-11-13T19:53:56,122 WARN [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/.initializing 2024-11-13T19:53:56,122 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/WALs/54d0784d1943,35115,1731527634648 2024-11-13T19:53:56,131 INFO [master/54d0784d1943:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:53:56,151 INFO [master/54d0784d1943:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C35115%2C1731527634648, suffix=, logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/WALs/54d0784d1943,35115,1731527634648, archiveDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/oldWALs, maxLogs=10 2024-11-13T19:53:56,195 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/WALs/54d0784d1943,35115,1731527634648/54d0784d1943%2C35115%2C1731527634648.1731527636158, exclude list is [], retry=0 2024-11-13T19:53:56,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:53:56,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:53:56,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:53:56,220 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:53:56,223 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T19:53:56,261 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-13T19:53:56,344 INFO [master/54d0784d1943:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/WALs/54d0784d1943,35115,1731527634648/54d0784d1943%2C35115%2C1731527634648.1731527636158 2024-11-13T19:53:56,345 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605)] 2024-11-13T19:53:56,345 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:53:56,346 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:53:56,349 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,349 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,391 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T19:53:56,424 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:56,427 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:56,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T19:53:56,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:56,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:53:56,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T19:53:56,437 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:56,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:53:56,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T19:53:56,443 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:56,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:53:56,444 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,449 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,452 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,459 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,460 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,464 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T19:53:56,468 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:53:56,474 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:53:56,476 INFO [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73790774, jitterRate=0.09956821799278259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T19:53:56,482 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731527636363Initializing all the Stores at 1731527636365 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731527636365Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527636368 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527636368Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527636368Cleaning up temporary data from old regions at 1731527636460 (+92 ms)Region opened successfully at 1731527636482 (+22 ms) 2024-11-13T19:53:56,490 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T19:53:56,542 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ff8426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54d0784d1943/172.17.0.2:0 2024-11-13T19:53:56,588 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T19:53:56,604 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T19:53:56,605 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T19:53:56,609 INFO [master/54d0784d1943:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T19:53:56,610 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T19:53:56,617 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T19:53:56,617 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T19:53:56,656 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T19:53:56,666 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T19:53:56,669 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T19:53:56,672 INFO [master/54d0784d1943:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T19:53:56,675 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T19:53:56,677 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T19:53:56,680 INFO [master/54d0784d1943:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T19:53:56,685 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T19:53:56,687 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T19:53:56,688 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T19:53:56,690 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T19:53:56,709 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T19:53:56,711 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T19:53:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:53:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:53:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:53:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:53:56,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,725 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=54d0784d1943,35115,1731527634648, sessionid=0x100eda9e0200000, setting cluster-up flag (Was=false) 2024-11-13T19:53:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,754 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T19:53:56,756 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54d0784d1943,35115,1731527634648 2024-11-13T19:53:56,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,762 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:56,769 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T19:53:56,771 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=54d0784d1943,35115,1731527634648 2024-11-13T19:53:56,778 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T19:53:56,853 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(746): ClusterId : 697cb3e1-bdea-4df8-92f0-0da5980bf2ec 2024-11-13T19:53:56,857 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T19:53:56,858 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(746): ClusterId : 697cb3e1-bdea-4df8-92f0-0da5980bf2ec 2024-11-13T19:53:56,858 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T19:53:56,859 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(746): ClusterId : 697cb3e1-bdea-4df8-92f0-0da5980bf2ec 2024-11-13T19:53:56,859 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T19:53:56,865 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T19:53:56,865 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T19:53:56,865 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T19:53:56,865 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T19:53:56,866 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T19:53:56,866 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T19:53:56,874 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T19:53:56,874 DEBUG [RS:0;54d0784d1943:46741 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@762d00c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54d0784d1943/172.17.0.2:0 2024-11-13T19:53:56,873 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T19:53:56,879 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T19:53:56,880 DEBUG [RS:1;54d0784d1943:36813 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67fbaa23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54d0784d1943/172.17.0.2:0 2024-11-13T19:53:56,884 DEBUG [RS:2;54d0784d1943:35943 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fc56f01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=54d0784d1943/172.17.0.2:0 2024-11-13T19:53:56,896 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;54d0784d1943:46741 2024-11-13T19:53:56,901 INFO [RS:0;54d0784d1943:46741 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T19:53:56,901 INFO [RS:0;54d0784d1943:46741 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T19:53:56,901 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T19:53:56,904 DEBUG [RS:2;54d0784d1943:35943 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;54d0784d1943:35943 2024-11-13T19:53:56,904 INFO [RS:2;54d0784d1943:35943 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T19:53:56,904 INFO [RS:2;54d0784d1943:35943 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T19:53:56,904 DEBUG [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T19:53:56,905 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;54d0784d1943:36813 2024-11-13T19:53:56,905 INFO [RS:1;54d0784d1943:36813 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T19:53:56,905 INFO [RS:1;54d0784d1943:36813 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T19:53:56,905 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T19:53:56,908 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(2659): reportForDuty to master=54d0784d1943,35115,1731527634648 with port=46741, startcode=1731527635545 2024-11-13T19:53:56,909 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(2659): reportForDuty to master=54d0784d1943,35115,1731527634648 with port=35943, startcode=1731527635712 2024-11-13T19:53:56,909 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T19:53:56,911 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(2659): reportForDuty to master=54d0784d1943,35115,1731527634648 with port=36813, startcode=1731527635668 2024-11-13T19:53:56,923 INFO [master/54d0784d1943:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T19:53:56,925 DEBUG [RS:1;54d0784d1943:36813 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T19:53:56,925 DEBUG [RS:2;54d0784d1943:35943 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T19:53:56,925 DEBUG [RS:0;54d0784d1943:46741 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T19:53:56,933 INFO [master/54d0784d1943:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T19:53:56,940 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 54d0784d1943,35115,1731527634648 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T19:53:56,950 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:53:56,950 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:53:56,950 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:53:56,950 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/54d0784d1943:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:53:56,950 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/54d0784d1943:0, corePoolSize=10, maxPoolSize=10 2024-11-13T19:53:56,951 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:56,951 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/54d0784d1943:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:53:56,951 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:56,984 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:53:56,985 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T19:53:56,999 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51773, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T19:53:56,999 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731527666999 2024-11-13T19:53:57,000 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T19:53:57,001 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T19:53:57,002 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42665, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T19:53:57,003 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T19:53:57,004 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:57,004 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T19:53:57,007 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T19:53:57,007 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:53:57,008 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T19:53:57,008 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T19:53:57,008 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T19:53:57,009 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,016 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:53:57,017 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T19:53:57,024 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T19:53:57,026 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T19:53:57,026 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T19:53:57,049 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T19:53:57,050 INFO [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T19:53:57,051 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T19:53:57,051 WARN [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T19:53:57,051 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T19:53:57,051 WARN [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T19:53:57,053 DEBUG [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T19:53:57,054 WARN [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T19:53:57,061 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.large.0-1731527637051,5,FailOnTimeoutGroup] 2024-11-13T19:53:57,063 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.small.0-1731527637062,5,FailOnTimeoutGroup] 2024-11-13T19:53:57,063 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,063 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T19:53:57,065 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,065 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741831_1007 (size=1321) 2024-11-13T19:53:57,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741831_1007 (size=1321) 2024-11-13T19:53:57,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741831_1007 (size=1321) 2024-11-13T19:53:57,107 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T19:53:57,108 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976 2024-11-13T19:53:57,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741832_1008 (size=32) 2024-11-13T19:53:57,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741832_1008 (size=32) 2024-11-13T19:53:57,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741832_1008 (size=32) 2024-11-13T19:53:57,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:53:57,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T19:53:57,138 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T19:53:57,138 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:57,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:57,139 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T19:53:57,142 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T19:53:57,142 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:57,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:57,143 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T19:53:57,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T19:53:57,146 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:57,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:57,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T19:53:57,153 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(2659): reportForDuty to master=54d0784d1943,35115,1731527634648 with port=36813, startcode=1731527635668 2024-11-13T19:53:57,154 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(2659): reportForDuty to master=54d0784d1943,35115,1731527634648 with port=46741, startcode=1731527635545 2024-11-13T19:53:57,155 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 54d0784d1943,36813,1731527635668 2024-11-13T19:53:57,158 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(2659): reportForDuty to master=54d0784d1943,35115,1731527634648 with port=35943, startcode=1731527635712 2024-11-13T19:53:57,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T19:53:57,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] master.ServerManager(517): Registering regionserver=54d0784d1943,36813,1731527635668 2024-11-13T19:53:57,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:57,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:57,162 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T19:53:57,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740 2024-11-13T19:53:57,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740 2024-11-13T19:53:57,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T19:53:57,170 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T19:53:57,171 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T19:53:57,173 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 54d0784d1943,46741,1731527635545 2024-11-13T19:53:57,173 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] master.ServerManager(517): Registering regionserver=54d0784d1943,46741,1731527635545 2024-11-13T19:53:57,173 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976 2024-11-13T19:53:57,173 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39891 2024-11-13T19:53:57,174 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T19:53:57,181 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976 2024-11-13T19:53:57,181 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39891 2024-11-13T19:53:57,181 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T19:53:57,182 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 54d0784d1943,35943,1731527635712 2024-11-13T19:53:57,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T19:53:57,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:53:57,182 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] master.ServerManager(517): Registering regionserver=54d0784d1943,35943,1731527635712 2024-11-13T19:53:57,186 DEBUG [RS:1;54d0784d1943:36813 {}] zookeeper.ZKUtil(111): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/54d0784d1943,36813,1731527635668 2024-11-13T19:53:57,186 WARN [RS:1;54d0784d1943:36813 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:53:57,186 INFO [RS:1;54d0784d1943:36813 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:53:57,186 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668 2024-11-13T19:53:57,192 DEBUG [RS:0;54d0784d1943:46741 {}] zookeeper.ZKUtil(111): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/54d0784d1943,46741,1731527635545 2024-11-13T19:53:57,192 WARN [RS:0;54d0784d1943:46741 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:53:57,193 INFO [RS:0;54d0784d1943:46741 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:53:57,193 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,46741,1731527635545 2024-11-13T19:53:57,194 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [54d0784d1943,36813,1731527635668] 2024-11-13T19:53:57,194 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [54d0784d1943,46741,1731527635545] 2024-11-13T19:53:57,195 DEBUG [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976 2024-11-13T19:53:57,195 DEBUG [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39891 2024-11-13T19:53:57,196 DEBUG [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T19:53:57,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:53:57,199 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:53:57,200 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [54d0784d1943,35943,1731527635712] 2024-11-13T19:53:57,201 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66511887, jitterRate=-0.008895650506019592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T19:53:57,202 DEBUG [RS:2;54d0784d1943:35943 {}] zookeeper.ZKUtil(111): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/54d0784d1943,35943,1731527635712 2024-11-13T19:53:57,202 WARN [RS:2;54d0784d1943:35943 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:53:57,202 INFO [RS:2;54d0784d1943:35943 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:53:57,203 DEBUG [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,35943,1731527635712 2024-11-13T19:53:57,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731527637132Initializing all the Stores at 1731527637134 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731527637134Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731527637135 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527637135Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731527637135Cleaning up temporary data from old regions at 1731527637170 (+35 ms)Region opened successfully at 1731527637204 (+34 ms) 2024-11-13T19:53:57,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T19:53:57,205 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T19:53:57,205 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T19:53:57,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 1 ms 2024-11-13T19:53:57,206 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T19:53:57,207 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T19:53:57,208 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731527637205Disabling compacts and flushes for region at 1731527637205Disabling writes for close at 1731527637206 (+1 ms)Writing region close event to WAL at 1731527637207 (+1 ms)Closed at 1731527637207 2024-11-13T19:53:57,215 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:53:57,215 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T19:53:57,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T19:53:57,232 INFO [RS:2;54d0784d1943:35943 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T19:53:57,233 INFO [RS:0;54d0784d1943:46741 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T19:53:57,233 INFO [RS:1;54d0784d1943:36813 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T19:53:57,244 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T19:53:57,255 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T19:53:57,260 INFO [RS:2;54d0784d1943:35943 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T19:53:57,262 INFO [RS:1;54d0784d1943:36813 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T19:53:57,273 INFO [RS:2;54d0784d1943:35943 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T19:53:57,273 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,277 INFO [RS:1;54d0784d1943:36813 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T19:53:57,278 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,278 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T19:53:57,282 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T19:53:57,283 INFO [RS:0;54d0784d1943:46741 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T19:53:57,285 INFO [RS:0;54d0784d1943:46741 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T19:53:57,285 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,285 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T19:53:57,286 INFO [RS:1;54d0784d1943:36813 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T19:53:57,286 INFO [RS:2;54d0784d1943:35943 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T19:53:57,288 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,288 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,288 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,288 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,288 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,288 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,288 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,289 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:53:57,289 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,289 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,289 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,289 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,289 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,289 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,290 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:53:57,290 DEBUG [RS:1;54d0784d1943:36813 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:53:57,290 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,290 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,290 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,290 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,290 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,291 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:53:57,291 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,291 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,291 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,291 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,291 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,291 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,292 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:53:57,292 DEBUG [RS:2;54d0784d1943:35943 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:53:57,294 INFO [RS:0;54d0784d1943:46741 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T19:53:57,294 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,295 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,295 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,295 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,295 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,295 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,295 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:53:57,297 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,297 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,297 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,298 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,298 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,298 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/54d0784d1943:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:53:57,298 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:53:57,298 DEBUG [RS:0;54d0784d1943:46741 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/54d0784d1943:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:53:57,321 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,321 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,321 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,321 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,322 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,322 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,36813,1731527635668-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:53:57,322 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,322 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,322 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,322 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,46741,1731527635545-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,323 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35943,1731527635712-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:53:57,367 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T19:53:57,370 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35943,1731527635712-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,371 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T19:53:57,371 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,46741,1731527635545-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,371 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,371 INFO [RS:0;54d0784d1943:46741 {}] regionserver.Replication(171): 54d0784d1943,46741,1731527635545 started 2024-11-13T19:53:57,372 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,373 INFO [RS:2;54d0784d1943:35943 {}] regionserver.Replication(171): 54d0784d1943,35943,1731527635712 started 2024-11-13T19:53:57,385 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T19:53:57,385 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,36813,1731527635668-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,386 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,389 INFO [RS:1;54d0784d1943:36813 {}] regionserver.Replication(171): 54d0784d1943,36813,1731527635668 started 2024-11-13T19:53:57,407 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,408 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1482): Serving as 54d0784d1943,46741,1731527635545, RpcServer on 54d0784d1943/172.17.0.2:46741, sessionid=0x100eda9e0200001 2024-11-13T19:53:57,408 WARN [54d0784d1943:35115 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T19:53:57,409 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,409 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(1482): Serving as 54d0784d1943,35943,1731527635712, RpcServer on 54d0784d1943/172.17.0.2:35943, sessionid=0x100eda9e0200003 2024-11-13T19:53:57,410 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T19:53:57,410 DEBUG [RS:2;54d0784d1943:35943 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 54d0784d1943,35943,1731527635712 2024-11-13T19:53:57,414 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:57,417 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T19:53:57,417 DEBUG [RS:0;54d0784d1943:46741 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 54d0784d1943,46741,1731527635545 2024-11-13T19:53:57,417 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,46741,1731527635545' 2024-11-13T19:53:57,417 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,35943,1731527635712' 2024-11-13T19:53:57,417 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T19:53:57,417 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T19:53:57,418 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T19:53:57,419 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T19:53:57,419 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T19:53:57,419 DEBUG [RS:0;54d0784d1943:46741 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 54d0784d1943,46741,1731527635545 2024-11-13T19:53:57,419 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,46741,1731527635545' 2024-11-13T19:53:57,419 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T19:53:57,420 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1482): Serving as 54d0784d1943,36813,1731527635668, RpcServer on 54d0784d1943/172.17.0.2:36813, sessionid=0x100eda9e0200002 2024-11-13T19:53:57,420 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T19:53:57,420 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T19:53:57,420 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T19:53:57,420 DEBUG [RS:1;54d0784d1943:36813 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 54d0784d1943,36813,1731527635668 2024-11-13T19:53:57,421 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,36813,1731527635668' 2024-11-13T19:53:57,421 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T19:53:57,422 DEBUG [RS:0;54d0784d1943:46741 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T19:53:57,422 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T19:53:57,422 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T19:53:57,422 DEBUG [RS:2;54d0784d1943:35943 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 54d0784d1943,35943,1731527635712 2024-11-13T19:53:57,422 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T19:53:57,422 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,35943,1731527635712' 2024-11-13T19:53:57,422 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T19:53:57,423 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T19:53:57,423 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T19:53:57,423 DEBUG [RS:1;54d0784d1943:36813 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 54d0784d1943,36813,1731527635668 2024-11-13T19:53:57,423 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '54d0784d1943,36813,1731527635668' 2024-11-13T19:53:57,423 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T19:53:57,423 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T19:53:57,424 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T19:53:57,424 DEBUG [RS:2;54d0784d1943:35943 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T19:53:57,424 INFO [RS:2;54d0784d1943:35943 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T19:53:57,424 INFO [RS:2;54d0784d1943:35943 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T19:53:57,425 DEBUG [RS:1;54d0784d1943:36813 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T19:53:57,425 INFO [RS:1;54d0784d1943:36813 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T19:53:57,425 INFO [RS:1;54d0784d1943:36813 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T19:53:57,422 INFO [RS:0;54d0784d1943:46741 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T19:53:57,425 INFO [RS:0;54d0784d1943:46741 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T19:53:57,531 INFO [RS:1;54d0784d1943:36813 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:53:57,531 INFO [RS:0;54d0784d1943:46741 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:53:57,531 INFO [RS:2;54d0784d1943:35943 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:53:57,535 INFO [RS:0;54d0784d1943:46741 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C46741%2C1731527635545, suffix=, logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,46741,1731527635545, archiveDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs, maxLogs=32 2024-11-13T19:53:57,535 INFO [RS:1;54d0784d1943:36813 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C36813%2C1731527635668, suffix=, logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668, archiveDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs, maxLogs=32 2024-11-13T19:53:57,538 INFO [RS:2;54d0784d1943:35943 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C35943%2C1731527635712, suffix=, logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,35943,1731527635712, archiveDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs, maxLogs=32 2024-11-13T19:53:57,567 DEBUG [RS:0;54d0784d1943:46741 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,46741,1731527635545/54d0784d1943%2C46741%2C1731527635545.1731527637542, exclude list is [], retry=0 2024-11-13T19:53:57,568 DEBUG [RS:2;54d0784d1943:35943 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,35943,1731527635712/54d0784d1943%2C35943%2C1731527635712.1731527637542, exclude list is [], retry=0 2024-11-13T19:53:57,568 DEBUG [RS:1;54d0784d1943:36813 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668/54d0784d1943%2C36813%2C1731527635668.1731527637542, exclude list is [], retry=0 2024-11-13T19:53:57,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:53:57,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:53:57,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:53:57,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:53:57,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:53:57,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:53:57,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:53:57,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:53:57,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:53:57,646 INFO [RS:0;54d0784d1943:46741 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,46741,1731527635545/54d0784d1943%2C46741%2C1731527635545.1731527637542 2024-11-13T19:53:57,649 INFO [RS:2;54d0784d1943:35943 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,35943,1731527635712/54d0784d1943%2C35943%2C1731527635712.1731527637542 2024-11-13T19:53:57,649 INFO [RS:1;54d0784d1943:36813 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668/54d0784d1943%2C36813%2C1731527635668.1731527637542 2024-11-13T19:53:57,652 DEBUG [RS:0;54d0784d1943:46741 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:53:57,661 DEBUG [RS:1;54d0784d1943:36813 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:53:57,662 DEBUG [RS:2;54d0784d1943:35943 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:53:57,918 DEBUG [54d0784d1943:35115 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T19:53:57,926 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(204): Hosts are {54d0784d1943=0} racks are {/default-rack=0} 2024-11-13T19:53:57,933 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T19:53:57,933 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T19:53:57,933 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T19:53:57,933 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T19:53:57,933 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T19:53:57,933 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T19:53:57,933 INFO [54d0784d1943:35115 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T19:53:57,933 INFO [54d0784d1943:35115 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T19:53:57,933 INFO [54d0784d1943:35115 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T19:53:57,933 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T19:53:57,943 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=54d0784d1943,46741,1731527635545 2024-11-13T19:53:57,949 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54d0784d1943,46741,1731527635545, state=OPENING 2024-11-13T19:53:57,954 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T19:53:57,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:57,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:57,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:57,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:53:57,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:57,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:57,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:57,958 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:57,959 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T19:53:57,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=54d0784d1943,46741,1731527635545}] 2024-11-13T19:53:58,140 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T19:53:58,142 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T19:53:58,156 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T19:53:58,157 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:53:58,157 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T19:53:58,161 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=54d0784d1943%2C46741%2C1731527635545.meta, suffix=.meta, logDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,46741,1731527635545, archiveDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs, maxLogs=32 2024-11-13T19:53:58,183 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,46741,1731527635545/54d0784d1943%2C46741%2C1731527635545.meta.1731527638163.meta, exclude list is [], retry=0 2024-11-13T19:53:58,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:53:58,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:53:58,188 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:53:58,227 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,46741,1731527635545/54d0784d1943%2C46741%2C1731527635545.meta.1731527638163.meta 2024-11-13T19:53:58,228 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:53:58,228 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:53:58,230 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T19:53:58,233 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T19:53:58,239 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T19:53:58,245 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T19:53:58,245 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:53:58,246 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T19:53:58,246 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T19:53:58,256 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T19:53:58,263 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T19:53:58,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:58,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:58,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T19:53:58,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T19:53:58,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:58,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:58,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T19:53:58,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T19:53:58,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:58,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:58,277 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T19:53:58,278 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T19:53:58,278 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:58,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:53:58,279 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T19:53:58,281 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740 2024-11-13T19:53:58,283 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740 2024-11-13T19:53:58,286 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T19:53:58,286 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T19:53:58,287 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T19:53:58,289 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T19:53:58,291 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61742812, jitterRate=-0.0799604058265686}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T19:53:58,291 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T19:53:58,293 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731527638246Writing region info on filesystem at 1731527638247 (+1 ms)Initializing all the Stores at 1731527638254 (+7 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731527638254Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731527638256 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527638256Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731527638256Cleaning up temporary data from old regions at 1731527638286 (+30 ms)Running coprocessor post-open hooks at 1731527638291 (+5 ms)Region opened successfully at 1731527638293 (+2 ms) 2024-11-13T19:53:58,301 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731527638130 2024-11-13T19:53:58,314 DEBUG [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T19:53:58,314 INFO [RS_OPEN_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T19:53:58,316 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,46741,1731527635545 2024-11-13T19:53:58,319 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 54d0784d1943,46741,1731527635545, state=OPEN 2024-11-13T19:53:58,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:53:58,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:53:58,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:53:58,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:58,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:58,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:58,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:53:58,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:53:58,324 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=54d0784d1943,46741,1731527635545 2024-11-13T19:53:58,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T19:53:58,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=54d0784d1943,46741,1731527635545 in 363 msec 2024-11-13T19:53:58,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T19:53:58,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1100 sec 2024-11-13T19:53:58,343 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:53:58,343 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T19:53:58,367 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:53:58,369 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,46741,1731527635545, seqNum=-1] 2024-11-13T19:53:58,398 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:53:58,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53877, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:53:58,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6110 sec 2024-11-13T19:53:58,428 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731527638428, completionTime=-1 2024-11-13T19:53:58,431 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T19:53:58,431 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T19:53:58,470 INFO [master/54d0784d1943:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-13T19:53:58,470 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731527698470 2024-11-13T19:53:58,470 INFO [master/54d0784d1943:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731527758470 2024-11-13T19:53:58,470 INFO [master/54d0784d1943:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 39 msec 2024-11-13T19:53:58,472 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T19:53:58,480 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35115,1731527634648-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:58,481 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35115,1731527634648-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:58,481 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35115,1731527634648-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:58,482 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-54d0784d1943:35115, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:58,483 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:58,483 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:58,491 DEBUG [master/54d0784d1943:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T19:53:58,549 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.756sec 2024-11-13T19:53:58,554 INFO [master/54d0784d1943:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T19:53:58,556 INFO [master/54d0784d1943:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T19:53:58,557 INFO [master/54d0784d1943:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T19:53:58,558 INFO [master/54d0784d1943:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T19:53:58,558 INFO [master/54d0784d1943:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T19:53:58,559 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35115,1731527634648-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:53:58,560 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35115,1731527634648-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T19:53:58,567 DEBUG [master/54d0784d1943:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T19:53:58,569 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T19:53:58,569 INFO [master/54d0784d1943:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=54d0784d1943,35115,1731527634648-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:53:58,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29bee39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:53:58,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 54d0784d1943,35115,-1 for getting cluster id 2024-11-13T19:53:58,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:53:58,616 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '697cb3e1-bdea-4df8-92f0-0da5980bf2ec' 2024-11-13T19:53:58,618 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:53:58,618 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "697cb3e1-bdea-4df8-92f0-0da5980bf2ec" 2024-11-13T19:53:58,619 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2795f715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:53:58,619 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [54d0784d1943,35115,-1] 2024-11-13T19:53:58,623 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:53:58,625 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:53:58,626 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:53:58,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d365f59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:53:58,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:53:58,638 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=54d0784d1943,46741,1731527635545, seqNum=-1] 2024-11-13T19:53:58,639 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:53:58,641 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:53:58,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=54d0784d1943,35115,1731527634648 2024-11-13T19:53:58,665 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:39891/hbase 2024-11-13T19:53:58,685 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=361, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=2885 2024-11-13T19:53:58,706 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:53:58,710 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:53:58,711 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:53:58,719 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65945516, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-65945516, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:53:58,743 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65945516/hregion-65945516.1731527638722, exclude list is [], retry=0 2024-11-13T19:53:58,750 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:53:58,753 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:53:58,758 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:53:58,809 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65945516/hregion-65945516.1731527638722 2024-11-13T19:53:58,810 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:53:58,810 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e2ebcf9d0aec3fe030aa332a14648150, NAME => 'testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:53:58,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741838_1014 (size=64) 2024-11-13T19:53:58,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741838_1014 (size=64) 2024-11-13T19:53:58,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741838_1014 (size=64) 2024-11-13T19:53:58,833 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:53:58,839 INFO [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:58,841 INFO [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2ebcf9d0aec3fe030aa332a14648150 columnFamilyName a 2024-11-13T19:53:58,842 DEBUG [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:58,843 INFO [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] regionserver.HStore(327): Store=e2ebcf9d0aec3fe030aa332a14648150/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:53:58,843 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:58,855 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:58,856 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:58,857 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:58,857 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:58,861 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:58,865 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:53:58,866 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2ebcf9d0aec3fe030aa332a14648150; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66522835, jitterRate=-0.008732512593269348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:53:58,869 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2ebcf9d0aec3fe030aa332a14648150: Writing region info on filesystem at 1731527638833Initializing all the Stores at 1731527638838 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527638838Cleaning up temporary data from old regions at 1731527638857 (+19 ms)Region opened successfully at 1731527638869 (+12 ms) 2024-11-13T19:53:58,869 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e2ebcf9d0aec3fe030aa332a14648150, disabling compactions & flushes 2024-11-13T19:53:58,869 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150. 2024-11-13T19:53:58,869 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150. 2024-11-13T19:53:58,869 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150. after waiting 0 ms 2024-11-13T19:53:58,869 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150. 2024-11-13T19:53:58,872 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150. 2024-11-13T19:53:58,872 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e2ebcf9d0aec3fe030aa332a14648150: Waiting for close lock at 1731527638869Disabling compacts and flushes for region at 1731527638869Disabling writes for close at 1731527638869Writing region close event to WAL at 1731527638871 (+2 ms)Closed at 1731527638871 2024-11-13T19:53:58,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741837_1013 (size=175) 2024-11-13T19:53:58,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741837_1013 (size=175) 2024-11-13T19:53:58,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741837_1013 (size=175) 2024-11-13T19:53:58,885 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:53:58,886 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65945516:(num 1731527638722) 2024-11-13T19:53:58,888 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T19:53:58,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741839_1015 (size=469) 2024-11-13T19:53:58,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741839_1015 (size=469) 2024-11-13T19:53:58,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741839_1015 (size=469) 2024-11-13T19:53:59,346 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T19:53:59,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741840_1016 (size=322) 2024-11-13T19:53:59,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741840_1016 (size=322) 2024-11-13T19:53:59,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741840_1016 (size=322) 2024-11-13T19:53:59,433 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1, size=469 (469bytes) 2024-11-13T19:53:59,434 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T19:53:59,434 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T19:53:59,434 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1 2024-11-13T19:53:59,442 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1 after 6ms 2024-11-13T19:53:59,493 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:53:59,494 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1 took 63ms 2024-11-13T19:53:59,500 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1 so closing down 2024-11-13T19:53:59,500 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:53:59,506 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T19:53:59,549 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T19:53:59,550 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:53:59,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741841_1017 (size=469) 2024-11-13T19:53:59,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741841_1017 (size=469) 2024-11-13T19:53:59,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741841_1017 (size=469) 2024-11-13T19:53:59,567 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T19:53:59,570 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002 2024-11-13T19:53:59,574 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T19:53:59,574 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1, journal: Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1, size=469 (469bytes) at 1731527639433Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1 so closing down at 1731527639500 (+67 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000001-wal-1.temp at 1731527639549 (+49 ms)3 split writer threads finished at 1731527639551 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731527639567 (+16 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002 at 1731527639570 (+3 ms)Processed 2 edits across 1 Regions in 75 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731527639574 (+4 ms) 2024-11-13T19:53:59,590 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2, size=322 (322bytes) 2024-11-13T19:53:59,591 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2 2024-11-13T19:53:59,592 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2 after 1ms 2024-11-13T19:53:59,639 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:53:59,639 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2 took 49ms 2024-11-13T19:53:59,643 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2 so closing down 2024-11-13T19:53:59,643 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:53:59,646 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T19:53:59,680 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T19:53:59,680 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:53:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741842_1018 (size=322) 2024-11-13T19:53:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741842_1018 (size=322) 2024-11-13T19:53:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741842_1018 (size=322) 2024-11-13T19:53:59,694 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T19:53:59,726 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:53:59,729 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-13T19:53:59,731 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T19:53:59,731 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2, journal: Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2, size=322 (322bytes) at 1731527639590Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2 so closing down at 1731527639643 (+53 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002-wal-2.temp at 1731527639680 (+37 ms)3 split writer threads finished at 1731527639680Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731527639694 (+14 ms)Processed 1 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731527639731 (+37 ms) 2024-11-13T19:53:59,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:53:59,733 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:53:59,747 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal.1731527639735, exclude list is [], retry=0 2024-11-13T19:53:59,751 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:53:59,752 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:53:59,753 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:53:59,781 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal.1731527639735 2024-11-13T19:53:59,782 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:53:59,782 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e2ebcf9d0aec3fe030aa332a14648150, NAME => 'testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:53:59,782 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:53:59,782 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,782 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,785 INFO [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,788 INFO [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e2ebcf9d0aec3fe030aa332a14648150 columnFamilyName a 2024-11-13T19:53:59,788 DEBUG [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:53:59,789 INFO [StoreOpener-e2ebcf9d0aec3fe030aa332a14648150-1 {}] regionserver.HStore(327): Store=e2ebcf9d0aec3fe030aa332a14648150/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:53:59,790 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,791 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,794 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,796 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002 2024-11-13T19:53:59,828 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:53:59,835 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002 2024-11-13T19:53:59,838 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e2ebcf9d0aec3fe030aa332a14648150 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T19:53:59,890 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/.tmp/a/efe504a366114757a8c3679790360066 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731527638886/Put/seqid=0 2024-11-13T19:53:59,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741844_1020 (size=5170) 2024-11-13T19:53:59,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741844_1020 (size=5170) 2024-11-13T19:53:59,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741844_1020 (size=5170) 2024-11-13T19:53:59,912 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/.tmp/a/efe504a366114757a8c3679790360066 2024-11-13T19:53:59,959 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/.tmp/a/efe504a366114757a8c3679790360066 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/a/efe504a366114757a8c3679790360066 2024-11-13T19:53:59,968 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/a/efe504a366114757a8c3679790360066, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T19:53:59,973 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e2ebcf9d0aec3fe030aa332a14648150 in 135ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T19:53:59,975 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/0000000000000000002 2024-11-13T19:53:59,976 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,976 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,979 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e2ebcf9d0aec3fe030aa332a14648150 2024-11-13T19:53:59,983 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/e2ebcf9d0aec3fe030aa332a14648150/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T19:53:59,984 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e2ebcf9d0aec3fe030aa332a14648150; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72619747, jitterRate=0.08211855590343475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:53:59,986 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e2ebcf9d0aec3fe030aa332a14648150: Writing region info on filesystem at 1731527639782Initializing all the Stores at 1731527639784 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527639784Obtaining lock to block concurrent updates at 1731527639838 (+54 ms)Preparing flush snapshotting stores in e2ebcf9d0aec3fe030aa332a14648150 at 1731527639838Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731527639840 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731527638707.e2ebcf9d0aec3fe030aa332a14648150. at 1731527639841 (+1 ms)Flushing e2ebcf9d0aec3fe030aa332a14648150/a: creating writer at 1731527639842 (+1 ms)Flushing e2ebcf9d0aec3fe030aa332a14648150/a: appending metadata at 1731527639879 (+37 ms)Flushing e2ebcf9d0aec3fe030aa332a14648150/a: closing flushed file at 1731527639882 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b16a604: reopening flushed file at 1731527639957 (+75 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e2ebcf9d0aec3fe030aa332a14648150 in 135ms, sequenceid=2, compaction requested=false; wal=null at 1731527639974 (+17 ms)Cleaning up temporary data from old regions at 1731527639976 (+2 ms)Region opened successfully at 1731527639986 (+10 ms) 2024-11-13T19:54:00,022 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=373 (was 361) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:41674 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:44424 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:48066 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:48046 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:41658 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:39891/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:44586 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=2823 (was 2885) 2024-11-13T19:54:00,039 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=2818 2024-11-13T19:54:00,055 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:00,057 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:00,058 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:00,062 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82433787, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-82433787, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:00,077 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82433787/hregion-82433787.1731527640063, exclude list is [], retry=0 2024-11-13T19:54:00,081 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:00,082 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:00,083 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:00,126 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82433787/hregion-82433787.1731527640063 2024-11-13T19:54:00,127 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:00,127 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 974537ab97bc2598286a9e29c1c4c1ee, NAME => 'testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:00,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741846_1022 (size=64) 2024-11-13T19:54:00,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741846_1022 (size=64) 2024-11-13T19:54:00,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741846_1022 (size=64) 2024-11-13T19:54:00,148 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:00,150 INFO [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,153 INFO [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974537ab97bc2598286a9e29c1c4c1ee columnFamilyName a 2024-11-13T19:54:00,153 DEBUG [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:00,154 INFO [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] regionserver.HStore(327): Store=974537ab97bc2598286a9e29c1c4c1ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:00,154 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,155 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,156 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,157 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,157 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,160 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,164 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:00,164 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 974537ab97bc2598286a9e29c1c4c1ee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68814280, jitterRate=0.025412678718566895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:54:00,165 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 974537ab97bc2598286a9e29c1c4c1ee: Writing region info on filesystem at 1731527640148Initializing all the Stores at 1731527640150 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527640150Cleaning up temporary data from old regions at 1731527640157 (+7 ms)Region opened successfully at 1731527640165 (+8 ms) 2024-11-13T19:54:00,165 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 974537ab97bc2598286a9e29c1c4c1ee, disabling compactions & flushes 2024-11-13T19:54:00,165 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee. 2024-11-13T19:54:00,165 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee. 2024-11-13T19:54:00,165 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee. after waiting 0 ms 2024-11-13T19:54:00,165 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee. 2024-11-13T19:54:00,165 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee. 2024-11-13T19:54:00,166 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 974537ab97bc2598286a9e29c1c4c1ee: Waiting for close lock at 1731527640165Disabling compacts and flushes for region at 1731527640165Disabling writes for close at 1731527640165Writing region close event to WAL at 1731527640165Closed at 1731527640165 2024-11-13T19:54:00,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741845_1021 (size=175) 2024-11-13T19:54:00,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741845_1021 (size=175) 2024-11-13T19:54:00,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741845_1021 (size=175) 2024-11-13T19:54:00,177 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:00,177 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82433787:(num 1731527640063) 2024-11-13T19:54:00,178 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T19:54:00,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741847_1023 (size=469) 2024-11-13T19:54:00,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741847_1023 (size=469) 2024-11-13T19:54:00,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741847_1023 (size=469) 2024-11-13T19:54:00,237 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T19:54:00,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741848_1024 (size=322) 2024-11-13T19:54:00,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741848_1024 (size=322) 2024-11-13T19:54:00,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741848_1024 (size=322) 2024-11-13T19:54:00,302 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2, size=322 (322bytes) 2024-11-13T19:54:00,302 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2 2024-11-13T19:54:00,303 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2 after 1ms 2024-11-13T19:54:00,346 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:00,346 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2 took 44ms 2024-11-13T19:54:00,349 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2 so closing down 2024-11-13T19:54:00,349 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:00,352 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T19:54:00,393 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T19:54:00,394 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:00,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741849_1025 (size=322) 2024-11-13T19:54:00,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741849_1025 (size=322) 2024-11-13T19:54:00,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741849_1025 (size=322) 2024-11-13T19:54:00,405 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:00,407 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002 2024-11-13T19:54:00,407 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 61 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-13T19:54:00,407 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2, journal: Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2, size=322 (322bytes) at 1731527640302Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2 so closing down at 1731527640349 (+47 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002-wal-2.temp at 1731527640394 (+45 ms)3 split writer threads finished at 1731527640394Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731527640405 (+11 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002 at 1731527640407 (+2 ms)Processed 1 edits across 1 Regions in 61 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1731527640407 2024-11-13T19:54:00,422 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1, size=469 (469bytes) 2024-11-13T19:54:00,422 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1 2024-11-13T19:54:00,423 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1 after 1ms 2024-11-13T19:54:00,454 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:00,454 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1 took 32ms 2024-11-13T19:54:00,458 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1 so closing down 2024-11-13T19:54:00,458 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:00,461 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T19:54:00,497 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T19:54:00,497 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:00,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741850_1026 (size=469) 2024-11-13T19:54:00,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741850_1026 (size=469) 2024-11-13T19:54:00,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741850_1026 (size=469) 2024-11-13T19:54:00,508 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:00,546 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:00,549 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002, length=322 2024-11-13T19:54:00,552 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002 2024-11-13T19:54:00,552 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 97 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-13T19:54:00,552 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1, journal: Splitting hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1, size=469 (469bytes) at 1731527640422Finishing writing output for hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1 so closing down at 1731527640458 (+36 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000001-wal-1.temp at 1731527640497 (+39 ms)3 split writer threads finished at 1731527640497Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731527640508 (+11 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002 at 1731527640552 (+44 ms)Processed 2 edits across 1 Regions in 97 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1731527640552 2024-11-13T19:54:00,552 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:00,555 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:00,568 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal.1731527640556, exclude list is [], retry=0 2024-11-13T19:54:00,571 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:00,572 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:00,572 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:00,600 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal.1731527640556 2024-11-13T19:54:00,600 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:00,601 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 974537ab97bc2598286a9e29c1c4c1ee, NAME => 'testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:00,601 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:00,601 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,601 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,603 INFO [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,605 INFO [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 974537ab97bc2598286a9e29c1c4c1ee columnFamilyName a 2024-11-13T19:54:00,606 DEBUG [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:00,606 INFO [StoreOpener-974537ab97bc2598286a9e29c1c4c1ee-1 {}] regionserver.HStore(327): Store=974537ab97bc2598286a9e29c1c4c1ee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:00,607 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,607 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,610 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,611 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002 2024-11-13T19:54:00,650 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:00,653 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002 2024-11-13T19:54:00,653 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 974537ab97bc2598286a9e29c1c4c1ee 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T19:54:00,678 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/.tmp/a/da945fd013f343f991071b64dece6ed9 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731527640177/Put/seqid=0 2024-11-13T19:54:00,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741852_1028 (size=5170) 2024-11-13T19:54:00,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741852_1028 (size=5170) 2024-11-13T19:54:00,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741852_1028 (size=5170) 2024-11-13T19:54:00,690 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/.tmp/a/da945fd013f343f991071b64dece6ed9 2024-11-13T19:54:00,701 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/.tmp/a/da945fd013f343f991071b64dece6ed9 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/a/da945fd013f343f991071b64dece6ed9 2024-11-13T19:54:00,710 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/a/da945fd013f343f991071b64dece6ed9, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T19:54:00,710 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 974537ab97bc2598286a9e29c1c4c1ee in 57ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T19:54:00,711 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/0000000000000000002 2024-11-13T19:54:00,712 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,712 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,714 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 974537ab97bc2598286a9e29c1c4c1ee 2024-11-13T19:54:00,718 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/974537ab97bc2598286a9e29c1c4c1ee/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T19:54:00,719 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 974537ab97bc2598286a9e29c1c4c1ee; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73472943, jitterRate=0.0948321670293808}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:54:00,720 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 974537ab97bc2598286a9e29c1c4c1ee: Writing region info on filesystem at 1731527640601Initializing all the Stores at 1731527640603 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527640603Obtaining lock to block concurrent updates at 1731527640653 (+50 ms)Preparing flush snapshotting stores in 974537ab97bc2598286a9e29c1c4c1ee at 1731527640653Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731527640653Flushing stores of testReplayEditsWrittenIntoWAL,,1731527640056.974537ab97bc2598286a9e29c1c4c1ee. at 1731527640653Flushing 974537ab97bc2598286a9e29c1c4c1ee/a: creating writer at 1731527640654 (+1 ms)Flushing 974537ab97bc2598286a9e29c1c4c1ee/a: appending metadata at 1731527640676 (+22 ms)Flushing 974537ab97bc2598286a9e29c1c4c1ee/a: closing flushed file at 1731527640676Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28baa527: reopening flushed file at 1731527640700 (+24 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 974537ab97bc2598286a9e29c1c4c1ee in 57ms, sequenceid=2, compaction requested=false; wal=null at 1731527640710 (+10 ms)Cleaning up temporary data from old regions at 1731527640712 (+2 ms)Region opened successfully at 1731527640719 (+7 ms) 2024-11-13T19:54:00,740 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=383 (was 373) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:48046 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:48148 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:41728 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:44664 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=502 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=2801 (was 2818) 2024-11-13T19:54:00,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=383, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=2801 2024-11-13T19:54:00,772 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:00,774 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:00,775 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:00,778 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-76622942, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-76622942, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:00,798 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-76622942/hregion-76622942.1731527640779, exclude list is [], retry=0 2024-11-13T19:54:00,802 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:00,803 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:00,803 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:00,832 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-76622942/hregion-76622942.1731527640779 2024-11-13T19:54:00,832 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605)] 2024-11-13T19:54:00,833 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6a097fef86b737d282f88edc48d84619, NAME => 'testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:00,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741854_1030 (size=64) 2024-11-13T19:54:00,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741854_1030 (size=64) 2024-11-13T19:54:00,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741854_1030 (size=64) 2024-11-13T19:54:00,849 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:00,851 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,853 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a097fef86b737d282f88edc48d84619 columnFamilyName a 2024-11-13T19:54:00,853 DEBUG [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:00,854 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(327): Store=6a097fef86b737d282f88edc48d84619/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:00,854 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,856 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a097fef86b737d282f88edc48d84619 columnFamilyName b 2024-11-13T19:54:00,856 DEBUG [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:00,857 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(327): Store=6a097fef86b737d282f88edc48d84619/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:00,857 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,859 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a097fef86b737d282f88edc48d84619 columnFamilyName c 2024-11-13T19:54:00,859 DEBUG [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:00,860 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(327): Store=6a097fef86b737d282f88edc48d84619/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:00,860 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,861 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,862 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,863 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,864 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,865 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:00,866 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:00,870 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:00,870 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a097fef86b737d282f88edc48d84619; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60534171, jitterRate=-0.09797056019306183}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:00,871 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a097fef86b737d282f88edc48d84619: Writing region info on filesystem at 1731527640849Initializing all the Stores at 1731527640850 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527640850Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527640851 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527640851Cleaning up temporary data from old regions at 1731527640864 (+13 ms)Region opened successfully at 1731527640870 (+6 ms) 2024-11-13T19:54:00,871 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a097fef86b737d282f88edc48d84619, disabling compactions & flushes 2024-11-13T19:54:00,871 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:00,871 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:00,871 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. after waiting 0 ms 2024-11-13T19:54:00,871 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:00,872 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:00,872 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a097fef86b737d282f88edc48d84619: Waiting for close lock at 1731527640871Disabling compacts and flushes for region at 1731527640871Disabling writes for close at 1731527640871Writing region close event to WAL at 1731527640872 (+1 ms)Closed at 1731527640872 2024-11-13T19:54:00,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741853_1029 (size=175) 2024-11-13T19:54:00,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741853_1029 (size=175) 2024-11-13T19:54:00,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741853_1029 (size=175) 2024-11-13T19:54:00,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:00,879 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-76622942:(num 1731527640779) 2024-11-13T19:54:00,879 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:00,882 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:00,902 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883, exclude list is [], retry=0 2024-11-13T19:54:00,906 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:00,907 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:00,907 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:00,948 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 2024-11-13T19:54:00,949 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:01,175 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883, size=0 (0bytes) 2024-11-13T19:54:01,175 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 might be still open, length is 0 2024-11-13T19:54:01,175 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 2024-11-13T19:54:01,176 WARN [IPC Server handler 4 on default port 39891 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T19:54:01,177 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 after 1ms 2024-11-13T19:54:02,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:41756 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33527:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41756 dst: /127.0.0.1:33527 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33527 remote=/127.0.0.1:41756]. Total timeout mills is 60000, 58637 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:02,506 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:48192 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48192 dst: /127.0.0.1:43261 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:02,506 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:44690 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33541:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44690 dst: /127.0.0.1:33541 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:02,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741855_1032 (size=497775) 2024-11-13T19:54:02,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741855_1032 (size=497775) 2024-11-13T19:54:02,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741855_1032 (size=497775) 2024-11-13T19:54:03,224 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T19:54:03,756 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:54:05,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T19:54:05,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:05,134 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:54:05,134 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T19:54:05,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T19:54:05,135 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:05,136 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T19:54:05,136 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:05,178 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 after 4003ms 2024-11-13T19:54:05,220 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:05,222 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 took 4048ms 2024-11-13T19:54:05,228 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731527640883.temp 2024-11-13T19:54:05,289 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000000001-wal.1731527640883.temp 2024-11-13T19:54:05,381 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883; continuing. 2024-11-13T19:54:05,381 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 so closing down 2024-11-13T19:54:05,381 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:05,382 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:05,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741856_1033 (size=497783) 2024-11-13T19:54:05,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741856_1033 (size=497783) 2024-11-13T19:54:05,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741856_1033 (size=497783) 2024-11-13T19:54:05,398 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000000001-wal.1731527640883.temp (wrote 3002 edits, skipped 0 edits in 58 ms) 2024-11-13T19:54:05,404 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000000001-wal.1731527640883.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000003002 2024-11-13T19:54:05,404 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 182 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T19:54:05,404 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883, journal: Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883, size=0 (0bytes) at 1731527641175Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000000001-wal.1731527640883.temp at 1731527645289 (+4114 ms)Split 1024 edits, skipped 0 edits. at 1731527645326 (+37 ms)Split 2048 edits, skipped 0 edits. at 1731527645362 (+36 ms)Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 so closing down at 1731527645381 (+19 ms)3 split writer threads finished at 1731527645382 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000000001-wal.1731527640883.temp (wrote 3002 edits, skipped 0 edits in 58 ms) at 1731527645398 (+16 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000000001-wal.1731527640883.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000003002 at 1731527645404 (+6 ms)Processed 3002 edits across 1 Regions in 182 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883, size=0, length=0, corrupted=false, cancelled=false at 1731527645404 2024-11-13T19:54:05,420 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527640883 2024-11-13T19:54:05,426 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000003002 2024-11-13T19:54:05,426 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:05,437 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:05,454 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527645438, exclude list is [], retry=0 2024-11-13T19:54:05,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:05,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:05,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:05,515 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527645438 2024-11-13T19:54:05,519 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:05,520 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:05,529 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:05,537 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a097fef86b737d282f88edc48d84619 columnFamilyName a 2024-11-13T19:54:05,537 DEBUG [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:05,541 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(327): Store=6a097fef86b737d282f88edc48d84619/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:05,541 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:05,550 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a097fef86b737d282f88edc48d84619 columnFamilyName b 2024-11-13T19:54:05,550 DEBUG [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:05,551 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(327): Store=6a097fef86b737d282f88edc48d84619/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:05,552 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:05,554 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6a097fef86b737d282f88edc48d84619 columnFamilyName c 2024-11-13T19:54:05,554 DEBUG [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:05,554 INFO [StoreOpener-6a097fef86b737d282f88edc48d84619-1 {}] regionserver.HStore(327): Store=6a097fef86b737d282f88edc48d84619/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:05,555 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:05,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:05,568 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:05,569 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000003002 2024-11-13T19:54:05,617 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:05,659 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T19:54:06,140 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a097fef86b737d282f88edc48d84619 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T19:54:06,187 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/a/cfb2ab17d7f044119fd3a62d0b3fac42 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731527640954/Put/seqid=0 2024-11-13T19:54:06,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741858_1035 (size=50463) 2024-11-13T19:54:06,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741858_1035 (size=50463) 2024-11-13T19:54:06,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741858_1035 (size=50463) 2024-11-13T19:54:06,199 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/a/cfb2ab17d7f044119fd3a62d0b3fac42 2024-11-13T19:54:06,207 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/a/cfb2ab17d7f044119fd3a62d0b3fac42 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/a/cfb2ab17d7f044119fd3a62d0b3fac42 2024-11-13T19:54:06,215 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/a/cfb2ab17d7f044119fd3a62d0b3fac42, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T19:54:06,215 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 6a097fef86b737d282f88edc48d84619 in 76ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T19:54:06,232 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T19:54:06,232 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a097fef86b737d282f88edc48d84619 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T19:54:06,244 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/a/95737b6f2d0f42c2bc5022494e9949f8 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731527640984/Put/seqid=0 2024-11-13T19:54:06,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741859_1036 (size=20072) 2024-11-13T19:54:06,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741859_1036 (size=20072) 2024-11-13T19:54:06,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741859_1036 (size=20072) 2024-11-13T19:54:06,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/a/95737b6f2d0f42c2bc5022494e9949f8 2024-11-13T19:54:06,297 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/b/17b55d531c2f4d42b1eb581602e2b02f is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731527641022/Put/seqid=0 2024-11-13T19:54:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741860_1037 (size=35835) 2024-11-13T19:54:06,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741860_1037 (size=35835) 2024-11-13T19:54:06,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741860_1037 (size=35835) 2024-11-13T19:54:06,306 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/b/17b55d531c2f4d42b1eb581602e2b02f 2024-11-13T19:54:06,314 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/a/95737b6f2d0f42c2bc5022494e9949f8 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/a/95737b6f2d0f42c2bc5022494e9949f8 2024-11-13T19:54:06,321 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/a/95737b6f2d0f42c2bc5022494e9949f8, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T19:54:06,322 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/b/17b55d531c2f4d42b1eb581602e2b02f as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/b/17b55d531c2f4d42b1eb581602e2b02f 2024-11-13T19:54:06,329 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/b/17b55d531c2f4d42b1eb581602e2b02f, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T19:54:06,329 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6a097fef86b737d282f88edc48d84619 in 97ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T19:54:06,341 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T19:54:06,341 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a097fef86b737d282f88edc48d84619 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T19:54:06,348 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/b/328bd26972264fc6982173dba58b9777 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731527641049/Put/seqid=0 2024-11-13T19:54:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741861_1038 (size=35082) 2024-11-13T19:54:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741861_1038 (size=35082) 2024-11-13T19:54:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741861_1038 (size=35082) 2024-11-13T19:54:06,370 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/b/328bd26972264fc6982173dba58b9777 2024-11-13T19:54:06,398 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/c/c2025e848a6049fcac61864d0f53f64d is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731527641087/Put/seqid=0 2024-11-13T19:54:06,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741862_1039 (size=20825) 2024-11-13T19:54:06,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741862_1039 (size=20825) 2024-11-13T19:54:06,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741862_1039 (size=20825) 2024-11-13T19:54:06,409 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/c/c2025e848a6049fcac61864d0f53f64d 2024-11-13T19:54:06,419 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/b/328bd26972264fc6982173dba58b9777 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/b/328bd26972264fc6982173dba58b9777 2024-11-13T19:54:06,427 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/b/328bd26972264fc6982173dba58b9777, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T19:54:06,430 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/c/c2025e848a6049fcac61864d0f53f64d as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/c/c2025e848a6049fcac61864d0f53f64d 2024-11-13T19:54:06,438 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/c/c2025e848a6049fcac61864d0f53f64d, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T19:54:06,439 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 6a097fef86b737d282f88edc48d84619 in 98ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T19:54:06,454 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731527641140/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:06,459 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000003002 2024-11-13T19:54:06,459 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T19:54:06,460 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6a097fef86b737d282f88edc48d84619 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T19:54:06,471 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/c/7ca4c18fe78441a9b73eac7f89f01015 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731527641096/Put/seqid=0 2024-11-13T19:54:06,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741863_1040 (size=50301) 2024-11-13T19:54:06,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741863_1040 (size=50301) 2024-11-13T19:54:06,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741863_1040 (size=50301) 2024-11-13T19:54:06,494 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/c/7ca4c18fe78441a9b73eac7f89f01015 2024-11-13T19:54:06,505 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ca4c18fe78441a9b73eac7f89f01015 2024-11-13T19:54:06,507 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/.tmp/c/7ca4c18fe78441a9b73eac7f89f01015 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/c/7ca4c18fe78441a9b73eac7f89f01015 2024-11-13T19:54:06,514 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7ca4c18fe78441a9b73eac7f89f01015 2024-11-13T19:54:06,514 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/c/7ca4c18fe78441a9b73eac7f89f01015, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T19:54:06,515 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 6a097fef86b737d282f88edc48d84619 in 55ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T19:54:06,516 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/0000000000000003002 2024-11-13T19:54:06,517 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:06,517 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:06,518 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:54:06,520 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6a097fef86b737d282f88edc48d84619 2024-11-13T19:54:06,523 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenIntoWAL/6a097fef86b737d282f88edc48d84619/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T19:54:06,524 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6a097fef86b737d282f88edc48d84619; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71638854, jitterRate=0.06750211119651794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:54:06,525 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6a097fef86b737d282f88edc48d84619: Writing region info on filesystem at 1731527645520Initializing all the Stores at 1731527645525 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527645525Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527645529 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527645529Cleaning up temporary data from old regions at 1731527646517 (+988 ms)Region opened successfully at 1731527646525 (+8 ms) 2024-11-13T19:54:06,596 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6a097fef86b737d282f88edc48d84619, disabling compactions & flushes 2024-11-13T19:54:06,596 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:06,596 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:06,596 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. after waiting 0 ms 2024-11-13T19:54:06,596 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:06,599 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731527640772.6a097fef86b737d282f88edc48d84619. 2024-11-13T19:54:06,599 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6a097fef86b737d282f88edc48d84619: Waiting for close lock at 1731527646596Disabling compacts and flushes for region at 1731527646596Disabling writes for close at 1731527646596Writing region close event to WAL at 1731527646599 (+3 ms)Closed at 1731527646599 2024-11-13T19:54:06,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741857_1034 (size=175) 2024-11-13T19:54:06,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741857_1034 (size=175) 2024-11-13T19:54:06,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741857_1034 (size=175) 2024-11-13T19:54:06,611 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:06,611 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731527645438) 2024-11-13T19:54:06,630 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 383) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1158707414_22 at /127.0.0.1:56196 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1158707414_22 at /127.0.0.1:54050 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:32851 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1323050754) connection to localhost/127.0.0.1:32851 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1158707414_22 at /127.0.0.1:35778 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:39891 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1323050754) connection to localhost/127.0.0.1:46281 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1158707414_22 at /127.0.0.1:35798 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46281 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1323050754) connection to localhost/127.0.0.1:39891 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 502) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2804 (was 2801) - AvailableMemoryMB LEAK? - 2024-11-13T19:54:06,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=510, ProcessCount=11, AvailableMemoryMB=2797 2024-11-13T19:54:06,669 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:06,672 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:06,674 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:06,685 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-37119280, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-37119280, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:06,708 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-37119280/hregion-37119280.1731527646686, exclude list is [], retry=0 2024-11-13T19:54:06,714 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:06,718 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:06,718 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:06,768 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-37119280/hregion-37119280.1731527646686 2024-11-13T19:54:06,769 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:06,769 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 165ae7502b01bc56db701f0904d58929, NAME => 'test2727,,1731527646670.165ae7502b01bc56db701f0904d58929.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:06,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741865_1042 (size=43) 2024-11-13T19:54:06,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741865_1042 (size=43) 2024-11-13T19:54:06,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741865_1042 (size=43) 2024-11-13T19:54:06,784 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731527646670.165ae7502b01bc56db701f0904d58929.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:06,786 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,787 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 165ae7502b01bc56db701f0904d58929 columnFamilyName a 2024-11-13T19:54:06,788 DEBUG [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:06,788 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(327): Store=165ae7502b01bc56db701f0904d58929/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:06,789 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,791 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 165ae7502b01bc56db701f0904d58929 columnFamilyName b 2024-11-13T19:54:06,791 DEBUG [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:06,791 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(327): Store=165ae7502b01bc56db701f0904d58929/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:06,792 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,794 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 165ae7502b01bc56db701f0904d58929 columnFamilyName c 2024-11-13T19:54:06,794 DEBUG [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:06,798 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(327): Store=165ae7502b01bc56db701f0904d58929/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:06,798 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,804 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,804 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,805 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:06,807 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:06,810 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:06,811 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 165ae7502b01bc56db701f0904d58929; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75124524, jitterRate=0.1194426417350769}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:06,812 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 165ae7502b01bc56db701f0904d58929: Writing region info on filesystem at 1731527646784Initializing all the Stores at 1731527646785 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527646785Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527646785Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527646785Cleaning up temporary data from old regions at 1731527646804 (+19 ms)Region opened successfully at 1731527646812 (+8 ms) 2024-11-13T19:54:06,812 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 165ae7502b01bc56db701f0904d58929, disabling compactions & flushes 2024-11-13T19:54:06,812 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:06,813 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:06,813 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. after waiting 0 ms 2024-11-13T19:54:06,813 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:06,813 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:06,813 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 165ae7502b01bc56db701f0904d58929: Waiting for close lock at 1731527646812Disabling compacts and flushes for region at 1731527646812Disabling writes for close at 1731527646813 (+1 ms)Writing region close event to WAL at 1731527646813Closed at 1731527646813 2024-11-13T19:54:06,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741864_1041 (size=175) 2024-11-13T19:54:06,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741864_1041 (size=175) 2024-11-13T19:54:06,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741864_1041 (size=175) 2024-11-13T19:54:06,834 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:06,834 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-37119280:(num 1731527646686) 2024-11-13T19:54:06,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:06,837 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:06,851 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837, exclude list is [], retry=0 2024-11-13T19:54:06,854 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:06,855 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:06,855 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:06,887 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837 2024-11-13T19:54:06,888 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741866_1043 (size=371388) 2024-11-13T19:54:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741866_1043 (size=371388) 2024-11-13T19:54:07,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741866_1043 (size=371388) 2024-11-13T19:54:07,146 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837, size=362.7 K (371388bytes) 2024-11-13T19:54:07,146 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837 2024-11-13T19:54:07,153 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837 after 7ms 2024-11-13T19:54:07,235 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:07,240 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837 took 93ms 2024-11-13T19:54:07,251 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731527646837.temp 2024-11-13T19:54:07,296 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837 so closing down 2024-11-13T19:54:07,296 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:07,314 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000000001-wal.1731527646837.temp 2024-11-13T19:54:07,343 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:07,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741867_1044 (size=371388) 2024-11-13T19:54:07,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741867_1044 (size=371388) 2024-11-13T19:54:07,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741867_1044 (size=371388) 2024-11-13T19:54:07,361 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000000001-wal.1731527646837.temp (wrote 3000 edits, skipped 0 edits in 28 ms) 2024-11-13T19:54:07,363 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000000001-wal.1731527646837.temp to hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003000 2024-11-13T19:54:07,363 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 121 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-13T19:54:07,363 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837, journal: Splitting hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837, size=362.7 K (371388bytes) at 1731527647146Split 1024 edits, skipped 0 edits. at 1731527647263 (+117 ms)Split 2048 edits, skipped 0 edits. at 1731527647277 (+14 ms)Finishing writing output for hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837 so closing down at 1731527647296 (+19 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000000001-wal.1731527646837.temp at 1731527647314 (+18 ms)3 split writer threads finished at 1731527647343 (+29 ms)Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000000001-wal.1731527646837.temp (wrote 3000 edits, skipped 0 edits in 28 ms) at 1731527647361 (+18 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000000001-wal.1731527646837.temp to hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003000 at 1731527647363 (+2 ms)Processed 3000 edits across 1 Regions in 121 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1731527647363 2024-11-13T19:54:07,366 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527646837 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527646837 2024-11-13T19:54:07,367 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003000 2024-11-13T19:54:07,367 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:07,370 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:07,394 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371, exclude list is [], retry=0 2024-11-13T19:54:07,400 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:07,401 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:07,402 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:07,450 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371 2024-11-13T19:54:07,450 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605)] 2024-11-13T19:54:07,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741868_1045 (size=371515) 2024-11-13T19:54:07,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741868_1045 (size=371515) 2024-11-13T19:54:07,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741868_1045 (size=371515) 2024-11-13T19:54:07,648 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371, size=362.8 K (371515bytes) 2024-11-13T19:54:07,648 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371 2024-11-13T19:54:07,649 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371 after 1ms 2024-11-13T19:54:07,685 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:07,687 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371 took 39ms 2024-11-13T19:54:07,693 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731527647371.temp 2024-11-13T19:54:07,760 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371 so closing down 2024-11-13T19:54:07,761 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:07,786 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003001-wal.1731527647371.temp 2024-11-13T19:54:07,801 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741869_1046 (size=371515) 2024-11-13T19:54:07,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741869_1046 (size=371515) 2024-11-13T19:54:07,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741869_1046 (size=371515) 2024-11-13T19:54:07,830 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003001-wal.1731527647371.temp (wrote 3000 edits, skipped 0 edits in 14 ms) 2024-11-13T19:54:07,833 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003001-wal.1731527647371.temp to hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000006000 2024-11-13T19:54:07,833 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 146 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-13T19:54:07,833 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371, journal: Splitting hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371, size=362.8 K (371515bytes) at 1731527647648Split 1024 edits, skipped 0 edits. at 1731527647707 (+59 ms)Split 2048 edits, skipped 0 edits. at 1731527647751 (+44 ms)Finishing writing output for hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371 so closing down at 1731527647760 (+9 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003001-wal.1731527647371.temp at 1731527647786 (+26 ms)3 split writer threads finished at 1731527647801 (+15 ms)Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003001-wal.1731527647371.temp (wrote 3000 edits, skipped 0 edits in 14 ms) at 1731527647830 (+29 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003001-wal.1731527647371.temp to hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000006000 at 1731527647833 (+3 ms)Processed 3000 edits across 1 Regions in 146 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1731527647833 2024-11-13T19:54:07,842 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647371 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527647371 2024-11-13T19:54:07,853 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000006000 2024-11-13T19:54:07,854 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:07,862 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/test2727-manual,16010,1731527646668, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:07,887 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647863, exclude list is [], retry=0 2024-11-13T19:54:07,891 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:07,892 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:07,892 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:07,924 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731527646668/wal.1731527647863 2024-11-13T19:54:07,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:07,924 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 165ae7502b01bc56db701f0904d58929, NAME => 'test2727,,1731527646670.165ae7502b01bc56db701f0904d58929.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:07,925 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731527646670.165ae7502b01bc56db701f0904d58929.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:07,925 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,925 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,927 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,928 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 165ae7502b01bc56db701f0904d58929 columnFamilyName a 2024-11-13T19:54:07,928 DEBUG [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:07,929 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(327): Store=165ae7502b01bc56db701f0904d58929/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:07,929 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,930 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 165ae7502b01bc56db701f0904d58929 columnFamilyName b 2024-11-13T19:54:07,930 DEBUG [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:07,931 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(327): Store=165ae7502b01bc56db701f0904d58929/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:07,931 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,932 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 165ae7502b01bc56db701f0904d58929 columnFamilyName c 2024-11-13T19:54:07,932 DEBUG [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:07,933 INFO [StoreOpener-165ae7502b01bc56db701f0904d58929-1 {}] regionserver.HStore(327): Store=165ae7502b01bc56db701f0904d58929/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:07,933 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,934 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,936 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:07,937 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003000 2024-11-13T19:54:07,966 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:08,001 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003000 2024-11-13T19:54:08,003 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000006000 2024-11-13T19:54:08,033 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:08,058 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000006000 2024-11-13T19:54:08,058 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 165ae7502b01bc56db701f0904d58929 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T19:54:08,096 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/a/c0936e1a533b4adfaf3f36decd4d4c54 is 41, key is test2727/a:100/1731527647455/Put/seqid=0 2024-11-13T19:54:08,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741871_1048 (size=84227) 2024-11-13T19:54:08,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741871_1048 (size=84227) 2024-11-13T19:54:08,117 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/a/c0936e1a533b4adfaf3f36decd4d4c54 2024-11-13T19:54:08,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741871_1048 (size=84227) 2024-11-13T19:54:08,154 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/b/4b9566bb979840c5ada5a443da47ff14 is 41, key is test2727/b:100/1731527647501/Put/seqid=0 2024-11-13T19:54:08,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741872_1049 (size=84609) 2024-11-13T19:54:08,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741872_1049 (size=84609) 2024-11-13T19:54:08,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741872_1049 (size=84609) 2024-11-13T19:54:08,193 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/b/4b9566bb979840c5ada5a443da47ff14 2024-11-13T19:54:08,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/c/850893a7b3324f568b2006fe5bfcc04d is 41, key is test2727/c:100/1731527647561/Put/seqid=0 2024-11-13T19:54:08,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741873_1050 (size=84609) 2024-11-13T19:54:08,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741873_1050 (size=84609) 2024-11-13T19:54:08,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741873_1050 (size=84609) 2024-11-13T19:54:08,294 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/c/850893a7b3324f568b2006fe5bfcc04d 2024-11-13T19:54:08,309 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/a/c0936e1a533b4adfaf3f36decd4d4c54 as hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/a/c0936e1a533b4adfaf3f36decd4d4c54 2024-11-13T19:54:08,323 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/a/c0936e1a533b4adfaf3f36decd4d4c54, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T19:54:08,325 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/b/4b9566bb979840c5ada5a443da47ff14 as hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/b/4b9566bb979840c5ada5a443da47ff14 2024-11-13T19:54:08,335 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/b/4b9566bb979840c5ada5a443da47ff14, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T19:54:08,337 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/.tmp/c/850893a7b3324f568b2006fe5bfcc04d as hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/c/850893a7b3324f568b2006fe5bfcc04d 2024-11-13T19:54:08,345 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/c/850893a7b3324f568b2006fe5bfcc04d, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T19:54:08,345 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 165ae7502b01bc56db701f0904d58929 in 287ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T19:54:08,346 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000003000 2024-11-13T19:54:08,349 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/0000000000000006000 2024-11-13T19:54:08,350 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:08,350 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:08,351 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:08,356 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 165ae7502b01bc56db701f0904d58929 2024-11-13T19:54:08,360 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/test2727/165ae7502b01bc56db701f0904d58929/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T19:54:08,361 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 165ae7502b01bc56db701f0904d58929; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72192876, jitterRate=0.07575768232345581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:08,362 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 165ae7502b01bc56db701f0904d58929: Writing region info on filesystem at 1731527647925Initializing all the Stores at 1731527647926 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527647927 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527647927Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527647927Obtaining lock to block concurrent updates at 1731527648058 (+131 ms)Preparing flush snapshotting stores in 165ae7502b01bc56db701f0904d58929 at 1731527648058Finished memstore snapshotting test2727,,1731527646670.165ae7502b01bc56db701f0904d58929., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731527648059 (+1 ms)Flushing stores of test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. at 1731527648059Flushing 165ae7502b01bc56db701f0904d58929/a: creating writer at 1731527648059Flushing 165ae7502b01bc56db701f0904d58929/a: appending metadata at 1731527648095 (+36 ms)Flushing 165ae7502b01bc56db701f0904d58929/a: closing flushed file at 1731527648095Flushing 165ae7502b01bc56db701f0904d58929/b: creating writer at 1731527648125 (+30 ms)Flushing 165ae7502b01bc56db701f0904d58929/b: appending metadata at 1731527648152 (+27 ms)Flushing 165ae7502b01bc56db701f0904d58929/b: closing flushed file at 1731527648152Flushing 165ae7502b01bc56db701f0904d58929/c: creating writer at 1731527648201 (+49 ms)Flushing 165ae7502b01bc56db701f0904d58929/c: appending metadata at 1731527648229 (+28 ms)Flushing 165ae7502b01bc56db701f0904d58929/c: closing flushed file at 1731527648230 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5554dd7f: reopening flushed file at 1731527648306 (+76 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c9d5495: reopening flushed file at 1731527648324 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37fc1b3e: reopening flushed file at 1731527648336 (+12 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 165ae7502b01bc56db701f0904d58929 in 287ms, sequenceid=6000, compaction requested=false; wal=null at 1731527648345 (+9 ms)Cleaning up temporary data from old regions at 1731527648350 (+5 ms)Region opened successfully at 1731527648362 (+12 ms) 2024-11-13T19:54:08,364 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T19:54:08,364 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 165ae7502b01bc56db701f0904d58929, disabling compactions & flushes 2024-11-13T19:54:08,364 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:08,364 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:08,364 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. after waiting 0 ms 2024-11-13T19:54:08,364 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:08,373 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731527646670.165ae7502b01bc56db701f0904d58929. 2024-11-13T19:54:08,373 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 165ae7502b01bc56db701f0904d58929: Waiting for close lock at 1731527648364Disabling compacts and flushes for region at 1731527648364Disabling writes for close at 1731527648364Writing region close event to WAL at 1731527648373 (+9 ms)Closed at 1731527648373 2024-11-13T19:54:08,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741870_1047 (size=175) 2024-11-13T19:54:08,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741870_1047 (size=175) 2024-11-13T19:54:08,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741870_1047 (size=175) 2024-11-13T19:54:08,391 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:08,392 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731527647863) 2024-11-13T19:54:08,409 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=398 (was 399), OpenFileDescriptor=921 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=541 (was 510) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2444 (was 2797) 2024-11-13T19:54:08,423 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=398, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=541, ProcessCount=11, AvailableMemoryMB=2443 2024-11-13T19:54:08,439 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:08,447 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:08,448 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731527648447 2024-11-13T19:54:08,501 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447 2024-11-13T19:54:08,505 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:08,509 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2a76481afea043efba9312573b8fc329, NAME => 'testSequentialEditLogSeqNum,,1731527648439.2a76481afea043efba9312573b8fc329.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:08,510 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731527648439.2a76481afea043efba9312573b8fc329.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:08,510 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,510 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,511 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329 doesn't exist for region: 2a76481afea043efba9312573b8fc329 on table testSequentialEditLogSeqNum 2024-11-13T19:54:08,512 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 2a76481afea043efba9312573b8fc329 on table testSequentialEditLogSeqNum 2024-11-13T19:54:08,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741875_1052 (size=62) 2024-11-13T19:54:08,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741875_1052 (size=62) 2024-11-13T19:54:08,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741875_1052 (size=62) 2024-11-13T19:54:08,539 INFO [StoreOpener-2a76481afea043efba9312573b8fc329-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,541 INFO [StoreOpener-2a76481afea043efba9312573b8fc329-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2a76481afea043efba9312573b8fc329 columnFamilyName a 2024-11-13T19:54:08,541 DEBUG [StoreOpener-2a76481afea043efba9312573b8fc329-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:08,542 INFO [StoreOpener-2a76481afea043efba9312573b8fc329-1 {}] regionserver.HStore(327): Store=2a76481afea043efba9312573b8fc329/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:08,542 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,543 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,543 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,543 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,543 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,545 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2a76481afea043efba9312573b8fc329 2024-11-13T19:54:08,548 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:08,549 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2a76481afea043efba9312573b8fc329; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64071081, jitterRate=-0.045266494154930115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:54:08,549 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2a76481afea043efba9312573b8fc329: Writing region info on filesystem at 1731527648510Initializing all the Stores at 1731527648538 (+28 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527648538Cleaning up temporary data from old regions at 1731527648543 (+5 ms)Region opened successfully at 1731527648549 (+6 ms) 2024-11-13T19:54:08,562 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2a76481afea043efba9312573b8fc329 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T19:54:08,610 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/.tmp/a/efc3b45087084b978235737c37898477 is 81, key is testSequentialEditLogSeqNum/a:x0/1731527648550/Put/seqid=0 2024-11-13T19:54:08,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741876_1053 (size=5833) 2024-11-13T19:54:08,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741876_1053 (size=5833) 2024-11-13T19:54:08,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741876_1053 (size=5833) 2024-11-13T19:54:08,626 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/.tmp/a/efc3b45087084b978235737c37898477 2024-11-13T19:54:08,634 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/.tmp/a/efc3b45087084b978235737c37898477 as hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/a/efc3b45087084b978235737c37898477 2024-11-13T19:54:08,642 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/a/efc3b45087084b978235737c37898477, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T19:54:08,644 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 2a76481afea043efba9312573b8fc329 in 82ms, sequenceid=13, compaction requested=false 2024-11-13T19:54:08,644 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2a76481afea043efba9312573b8fc329: 2024-11-13T19:54:08,652 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T19:54:08,652 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T19:54:08,652 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T19:54:08,652 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T19:54:08,652 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T19:54:08,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741874_1051 (size=3256) 2024-11-13T19:54:08,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741874_1051 (size=3256) 2024-11-13T19:54:08,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741874_1051 (size=3256) 2024-11-13T19:54:08,680 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447, size=3.2 K (3256bytes) 2024-11-13T19:54:08,681 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447 2024-11-13T19:54:08,681 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447 after 0ms 2024-11-13T19:54:08,725 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:08,725 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447 took 45ms 2024-11-13T19:54:08,729 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447 so closing down 2024-11-13T19:54:08,730 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:08,731 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731527648447.temp 2024-11-13T19:54:08,778 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000003-wal.1731527648447.temp 2024-11-13T19:54:08,779 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:08,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741877_1054 (size=2650) 2024-11-13T19:54:08,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741877_1054 (size=2650) 2024-11-13T19:54:08,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741877_1054 (size=2650) 2024-11-13T19:54:08,792 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000003-wal.1731527648447.temp (wrote 15 edits, skipped 0 edits in 1 ms) 2024-11-13T19:54:08,794 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000003-wal.1731527648447.temp to hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000020 2024-11-13T19:54:08,794 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 69 ms; skipped=2; WAL=hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-13T19:54:08,794 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447, journal: Splitting hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447, size=3.2 K (3256bytes) at 1731527648681Finishing writing output for hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447 so closing down at 1731527648729 (+48 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000003-wal.1731527648447.temp at 1731527648778 (+49 ms)3 split writer threads finished at 1731527648779 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000003-wal.1731527648447.temp (wrote 15 edits, skipped 0 edits in 1 ms) at 1731527648792 (+13 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000003-wal.1731527648447.temp to hdfs://localhost:39891/hbase/data/default/testSequentialEditLogSeqNum/2a76481afea043efba9312573b8fc329/recovered.edits/0000000000000000020 at 1731527648794 (+2 ms)Processed 17 edits across 1 Regions in 69 ms; skipped=2; WAL=hdfs://localhost:39891/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731527648438/wal.1731527648447, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1731527648794 2024-11-13T19:54:08,809 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=403 (was 398) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:56282 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:35918 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=541 (was 541), ProcessCount=11 (was 11), AvailableMemoryMB=2528 (was 2443) - AvailableMemoryMB LEAK? - 2024-11-13T19:54:08,824 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=403, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=541, ProcessCount=11, AvailableMemoryMB=2522 2024-11-13T19:54:08,840 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:08,842 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:08,882 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:08,892 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99024388, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-99024388, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:08,913 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99024388/hregion-99024388.1731527648893, exclude list is [], retry=0 2024-11-13T19:54:08,922 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:08,922 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:08,923 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:08,956 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99024388/hregion-99024388.1731527648893 2024-11-13T19:54:08,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605)] 2024-11-13T19:54:08,956 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2c8d4200d9d3d33e3ec1da5526668c7e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741879_1056 (size=70) 2024-11-13T19:54:08,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741879_1056 (size=70) 2024-11-13T19:54:08,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741879_1056 (size=70) 2024-11-13T19:54:08,980 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:08,989 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:08,991 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName a 2024-11-13T19:54:08,992 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:08,992 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:08,992 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:08,994 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName b 2024-11-13T19:54:08,994 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:08,995 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:08,995 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:08,996 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName c 2024-11-13T19:54:08,996 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:08,997 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:08,997 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:08,998 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:08,998 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:08,999 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:08,999 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,000 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:09,001 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,003 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:09,004 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c8d4200d9d3d33e3ec1da5526668c7e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72389635, jitterRate=0.07868961989879608}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:09,004 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c8d4200d9d3d33e3ec1da5526668c7e: Writing region info on filesystem at 1731527648980Initializing all the Stores at 1731527648981 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527648982 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527648989 (+7 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527648989Cleaning up temporary data from old regions at 1731527648999 (+10 ms)Region opened successfully at 1731527649004 (+5 ms) 2024-11-13T19:54:09,004 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2c8d4200d9d3d33e3ec1da5526668c7e, disabling compactions & flushes 2024-11-13T19:54:09,004 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:09,005 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:09,005 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. after waiting 0 ms 2024-11-13T19:54:09,005 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:09,005 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:09,005 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2c8d4200d9d3d33e3ec1da5526668c7e: Waiting for close lock at 1731527649004Disabling compacts and flushes for region at 1731527649004Disabling writes for close at 1731527649005 (+1 ms)Writing region close event to WAL at 1731527649005Closed at 1731527649005 2024-11-13T19:54:09,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741878_1055 (size=175) 2024-11-13T19:54:09,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741878_1055 (size=175) 2024-11-13T19:54:09,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741878_1055 (size=175) 2024-11-13T19:54:09,010 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-99024388/hregion-99024388.1731527648893 not finished, retry = 0 2024-11-13T19:54:09,114 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:09,114 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99024388:(num 1731527648893) 2024-11-13T19:54:09,114 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:09,119 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:09,137 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119, exclude list is [], retry=0 2024-11-13T19:54:09,141 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:09,141 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:09,142 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:09,184 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 2024-11-13T19:54:09,184 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:09,184 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c8d4200d9d3d33e3ec1da5526668c7e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:09,185 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:09,185 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,185 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,193 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,199 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName a 2024-11-13T19:54:09,199 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:09,200 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:09,200 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,206 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName b 2024-11-13T19:54:09,206 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:09,209 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:09,209 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,216 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName c 2024-11-13T19:54:09,216 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:09,217 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:09,217 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,219 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,220 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,222 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,222 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,223 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:09,225 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:09,226 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c8d4200d9d3d33e3ec1da5526668c7e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70942240, jitterRate=0.05712175369262695}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:09,230 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c8d4200d9d3d33e3ec1da5526668c7e: Writing region info on filesystem at 1731527649185Initializing all the Stores at 1731527649193 (+8 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527649193Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527649193Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527649193Cleaning up temporary data from old regions at 1731527649222 (+29 ms)Region opened successfully at 1731527649230 (+8 ms) 2024-11-13T19:54:09,237 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731527649235/Put/seqid=0 2024-11-13T19:54:09,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741881_1058 (size=4826) 2024-11-13T19:54:09,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741881_1058 (size=4826) 2024-11-13T19:54:09,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741881_1058 (size=4826) 2024-11-13T19:54:09,663 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39891/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 2c8d4200d9d3d33e3ec1da5526668c7e/a 2024-11-13T19:54:09,672 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T19:54:09,672 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T19:54:09,672 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2c8d4200d9d3d33e3ec1da5526668c7e: 2024-11-13T19:54:09,675 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/a/d322351090a74a81a360f84957b6945c_SeqId_3_ 2024-11-13T19:54:09,676 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39891/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2c8d4200d9d3d33e3ec1da5526668c7e/a as hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/a/d322351090a74a81a360f84957b6945c_SeqId_3_ - updating store file list. 2024-11-13T19:54:09,683 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d322351090a74a81a360f84957b6945c_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:09,684 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/a/d322351090a74a81a360f84957b6945c_SeqId_3_ into 2c8d4200d9d3d33e3ec1da5526668c7e/a 2024-11-13T19:54:09,684 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39891/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2c8d4200d9d3d33e3ec1da5526668c7e/a (new location: hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/a/d322351090a74a81a360f84957b6945c_SeqId_3_) 2024-11-13T19:54:09,732 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119, size=0 (0bytes) 2024-11-13T19:54:09,732 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 might be still open, length is 0 2024-11-13T19:54:09,732 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 2024-11-13T19:54:09,733 WARN [IPC Server handler 3 on default port 39891 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T19:54:09,733 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 after 1ms 2024-11-13T19:54:11,488 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:54278 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33527:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54278 dst: /127.0.0.1:33527 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33527 remote=/127.0.0.1:54278]. Total timeout mills is 60000, 58204 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:11,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:36014 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33541:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36014 dst: /127.0.0.1:33541 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:11,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:56372 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56372 dst: /127.0.0.1:43261 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:11,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741880_1059 (size=606) 2024-11-13T19:54:11,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741880_1059 (size=606) 2024-11-13T19:54:13,734 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 after 4002ms 2024-11-13T19:54:13,764 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:13,764 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 took 4032ms 2024-11-13T19:54:13,767 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119; continuing. 2024-11-13T19:54:13,767 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 so closing down 2024-11-13T19:54:13,767 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:13,769 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731527649119.temp 2024-11-13T19:54:13,801 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005-wal.1731527649119.temp 2024-11-13T19:54:13,801 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:13,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741882_1060 (size=329) 2024-11-13T19:54:13,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741882_1060 (size=329) 2024-11-13T19:54:13,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741882_1060 (size=329) 2024-11-13T19:54:13,819 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005-wal.1731527649119.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:13,821 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005-wal.1731527649119.temp to hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005 2024-11-13T19:54:13,821 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 56 ms; skipped=1; WAL=hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T19:54:13,821 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119, journal: Splitting hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119, size=0 (0bytes) at 1731527649732Finishing writing output for hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 so closing down at 1731527653767 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005-wal.1731527649119.temp at 1731527653801 (+34 ms)3 split writer threads finished at 1731527653801Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005-wal.1731527649119.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731527653819 (+18 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005-wal.1731527649119.temp to hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005 at 1731527653821 (+2 ms)Processed 2 edits across 1 Regions in 56 ms; skipped=1; WAL=hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119, size=0, length=0, corrupted=false, cancelled=false at 1731527653821 2024-11-13T19:54:13,823 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527649119 2024-11-13T19:54:13,824 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005 2024-11-13T19:54:13,824 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:13,826 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:13,840 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527653827, exclude list is [], retry=0 2024-11-13T19:54:13,843 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:13,843 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:13,843 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:13,880 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527653827 2024-11-13T19:54:13,881 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:13,881 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2c8d4200d9d3d33e3ec1da5526668c7e, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:13,882 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:13,882 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,882 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,885 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,886 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName a 2024-11-13T19:54:13,886 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:13,896 DEBUG [StoreFileOpener-2c8d4200d9d3d33e3ec1da5526668c7e-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d322351090a74a81a360f84957b6945c_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:13,896 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/a/d322351090a74a81a360f84957b6945c_SeqId_3_ 2024-11-13T19:54:13,896 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:13,897 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,898 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName b 2024-11-13T19:54:13,898 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:13,898 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:13,898 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,899 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2c8d4200d9d3d33e3ec1da5526668c7e columnFamilyName c 2024-11-13T19:54:13,899 DEBUG [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:13,900 INFO [StoreOpener-2c8d4200d9d3d33e3ec1da5526668c7e-1 {}] regionserver.HStore(327): Store=2c8d4200d9d3d33e3ec1da5526668c7e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:13,900 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,902 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:13,903 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005 2024-11-13T19:54:13,944 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:13,946 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005 2024-11-13T19:54:13,946 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2c8d4200d9d3d33e3ec1da5526668c7e 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T19:54:13,975 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/.tmp/a/6ed8fe3cd11f454bb3d71a613747d7c5 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731527649691/Put/seqid=0 2024-11-13T19:54:14,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741884_1062 (size=5149) 2024-11-13T19:54:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741884_1062 (size=5149) 2024-11-13T19:54:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741884_1062 (size=5149) 2024-11-13T19:54:14,021 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/.tmp/a/6ed8fe3cd11f454bb3d71a613747d7c5 2024-11-13T19:54:14,035 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/.tmp/a/6ed8fe3cd11f454bb3d71a613747d7c5 as hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/a/6ed8fe3cd11f454bb3d71a613747d7c5 2024-11-13T19:54:14,042 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/a/6ed8fe3cd11f454bb3d71a613747d7c5, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T19:54:14,042 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2c8d4200d9d3d33e3ec1da5526668c7e in 96ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T19:54:14,043 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/0000000000000000005 2024-11-13T19:54:14,045 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:14,045 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:14,046 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:14,048 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2c8d4200d9d3d33e3ec1da5526668c7e 2024-11-13T19:54:14,050 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2c8d4200d9d3d33e3ec1da5526668c7e/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T19:54:14,052 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2c8d4200d9d3d33e3ec1da5526668c7e; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72264929, jitterRate=0.07683135569095612}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:14,052 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2c8d4200d9d3d33e3ec1da5526668c7e: Writing region info on filesystem at 1731527653882Initializing all the Stores at 1731527653884 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527653884Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527653884Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527653885 (+1 ms)Obtaining lock to block concurrent updates at 1731527653946 (+61 ms)Preparing flush snapshotting stores in 2c8d4200d9d3d33e3ec1da5526668c7e at 1731527653946Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731527653947 (+1 ms)Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. at 1731527653947Flushing 2c8d4200d9d3d33e3ec1da5526668c7e/a: creating writer at 1731527653947Flushing 2c8d4200d9d3d33e3ec1da5526668c7e/a: appending metadata at 1731527653974 (+27 ms)Flushing 2c8d4200d9d3d33e3ec1da5526668c7e/a: closing flushed file at 1731527653974Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56c24881: reopening flushed file at 1731527654034 (+60 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2c8d4200d9d3d33e3ec1da5526668c7e in 96ms, sequenceid=5, compaction requested=false; wal=null at 1731527654042 (+8 ms)Cleaning up temporary data from old regions at 1731527654045 (+3 ms)Region opened successfully at 1731527654052 (+7 ms) 2024-11-13T19:54:14,057 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2c8d4200d9d3d33e3ec1da5526668c7e, disabling compactions & flushes 2024-11-13T19:54:14,057 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:14,057 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:14,057 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. after waiting 0 ms 2024-11-13T19:54:14,057 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:14,058 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731527648841.2c8d4200d9d3d33e3ec1da5526668c7e. 2024-11-13T19:54:14,059 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2c8d4200d9d3d33e3ec1da5526668c7e: Waiting for close lock at 1731527654057Disabling compacts and flushes for region at 1731527654057Disabling writes for close at 1731527654057Writing region close event to WAL at 1731527654058 (+1 ms)Closed at 1731527654058 2024-11-13T19:54:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741883_1061 (size=175) 2024-11-13T19:54:14,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741883_1061 (size=175) 2024-11-13T19:54:14,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741883_1061 (size=175) 2024-11-13T19:54:14,079 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:14,079 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731527653827) 2024-11-13T19:54:14,099 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1580621981_22 at /127.0.0.1:42310 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:39891 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1580621981_22 at /127.0.0.1:41808 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1580621981_22 at /127.0.0.1:55562 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1323050754) connection to localhost/127.0.0.1:39891 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=529 (was 541), ProcessCount=11 (was 11), AvailableMemoryMB=2498 (was 2522) 2024-11-13T19:54:14,112 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=529, ProcessCount=11, AvailableMemoryMB=2496 2024-11-13T19:54:14,129 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:14,136 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:54:14,141 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 54d0784d1943,35115,1731527634648 2024-11-13T19:54:14,143 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36f0c023 2024-11-13T19:54:14,144 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:54:14,146 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:54:14,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:54:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T19:54:14,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:54:14,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T19:54:14,168 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:14,170 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:54:14,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:54:14,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741885_1063 (size=694) 2024-11-13T19:54:14,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741885_1063 (size=694) 2024-11-13T19:54:14,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741885_1063 (size=694) 2024-11-13T19:54:14,187 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab2bd533f2e133bf9e32cd3965a8e16d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976 2024-11-13T19:54:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741886_1064 (size=77) 2024-11-13T19:54:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741886_1064 (size=77) 2024-11-13T19:54:14,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741886_1064 (size=77) 2024-11-13T19:54:14,199 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:14,199 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing ab2bd533f2e133bf9e32cd3965a8e16d, disabling compactions & flushes 2024-11-13T19:54:14,199 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,199 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,199 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. after waiting 0 ms 2024-11-13T19:54:14,199 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,199 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,199 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab2bd533f2e133bf9e32cd3965a8e16d: Waiting for close lock at 1731527654199Disabling compacts and flushes for region at 1731527654199Disabling writes for close at 1731527654199Writing region close event to WAL at 1731527654199Closed at 1731527654199 2024-11-13T19:54:14,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:54:14,208 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731527654201"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731527654201"}]},"ts":"1731527654201"} 2024-11-13T19:54:14,211 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:54:14,212 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:54:14,215 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527654212"}]},"ts":"1731527654212"} 2024-11-13T19:54:14,219 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T19:54:14,219 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {54d0784d1943=0} racks are {/default-rack=0} 2024-11-13T19:54:14,221 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T19:54:14,221 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T19:54:14,221 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T19:54:14,221 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T19:54:14,221 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T19:54:14,221 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T19:54:14,221 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T19:54:14,221 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T19:54:14,221 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T19:54:14,221 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T19:54:14,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN}] 2024-11-13T19:54:14,224 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN 2024-11-13T19:54:14,225 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN; state=OFFLINE, location=54d0784d1943,36813,1731527635668; forceNewPlan=false, retain=false 2024-11-13T19:54:14,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:54:14,379 INFO [54d0784d1943:35115 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T19:54:14,380 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPENING, regionLocation=54d0784d1943,36813,1731527635668 2024-11-13T19:54:14,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN because future has completed 2024-11-13T19:54:14,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668}] 2024-11-13T19:54:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:54:14,539 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T19:54:14,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T19:54:14,547 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,547 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ab2bd533f2e133bf9e32cd3965a8e16d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:14,548 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,548 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:14,548 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,548 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,550 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,552 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf1 2024-11-13T19:54:14,552 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:14,553 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:14,553 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,555 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf2 2024-11-13T19:54:14,555 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:14,556 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:14,556 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,557 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,557 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,558 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,558 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,559 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T19:54:14,561 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,564 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:14,565 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ab2bd533f2e133bf9e32cd3965a8e16d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74451315, jitterRate=0.10941104590892792}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T19:54:14,565 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,566 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ab2bd533f2e133bf9e32cd3965a8e16d: Running coprocessor pre-open hook at 1731527654548Writing region info on filesystem at 1731527654548Initializing all the Stores at 1731527654549 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527654549Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527654550 (+1 ms)Cleaning up temporary data from old regions at 1731527654558 (+8 ms)Running coprocessor post-open hooks at 1731527654565 (+7 ms)Region opened successfully at 1731527654566 (+1 ms) 2024-11-13T19:54:14,568 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., pid=6, masterSystemTime=1731527654538 2024-11-13T19:54:14,571 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,572 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,572 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPEN, openSeqNum=2, regionLocation=54d0784d1943,36813,1731527635668 2024-11-13T19:54:14,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668 because future has completed 2024-11-13T19:54:14,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T19:54:14,580 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668 in 192 msec 2024-11-13T19:54:14,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T19:54:14,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN in 359 msec 2024-11-13T19:54:14,586 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:54:14,586 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731527654586"}]},"ts":"1731527654586"} 2024-11-13T19:54:14,588 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T19:54:14,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:54:14,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 438 msec 2024-11-13T19:54:14,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:54:14,805 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T19:54:14,805 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T19:54:14,806 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T19:54:14,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T19:54:14,814 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T19:54:14,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T19:54:14,831 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=2] 2024-11-13T19:54:14,833 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:54:14,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:54:14,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ab2bd533f2e133bf9e32cd3965a8e16d, source=54d0784d1943,36813,1731527635668, destination=54d0784d1943,46741,1731527635545, warming up region on 54d0784d1943,46741,1731527635545 2024-11-13T19:54:14,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ab2bd533f2e133bf9e32cd3965a8e16d, source=54d0784d1943,36813,1731527635668, destination=54d0784d1943,46741,1731527635545, running balancer 2024-11-13T19:54:14,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE 2024-11-13T19:54:14,856 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE 2024-11-13T19:54:14,859 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=CLOSING, regionLocation=54d0784d1943,36813,1731527635668 2024-11-13T19:54:14,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(7855): Warmup {ENCODED => ab2bd533f2e133bf9e32cd3965a8e16d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:14,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:14,862 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,863 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf1 2024-11-13T19:54:14,863 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:14,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE because future has completed 2024-11-13T19:54:14,864 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:54:14,865 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668}] 2024-11-13T19:54:14,865 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:14,865 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:14,869 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf2 2024-11-13T19:54:14,869 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:14,870 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(1722): Closing ab2bd533f2e133bf9e32cd3965a8e16d, disabling compactions & flushes 2024-11-13T19:54:14,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. after waiting 0 ms 2024-11-13T19:54:14,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] regionserver.HRegion(1676): Region close journal for ab2bd533f2e133bf9e32cd3965a8e16d: Waiting for close lock at 1731527654870Disabling compacts and flushes for region at 1731527654870Disabling writes for close at 1731527654870Writing region close event to WAL at 1731527654871 (+1 ms)Closed at 1731527654871 2024-11-13T19:54:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T19:54:15,023 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,023 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:54:15,024 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing ab2bd533f2e133bf9e32cd3965a8e16d, disabling compactions & flushes 2024-11-13T19:54:15,024 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,024 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,024 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. after waiting 0 ms 2024-11-13T19:54:15,024 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,024 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing ab2bd533f2e133bf9e32cd3965a8e16d 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T19:54:15,042 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf1/157880fb775746eba56e0f8460e4635d is 35, key is r1/cf1:q/1731527654836/Put/seqid=0 2024-11-13T19:54:15,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741887_1065 (size=4783) 2024-11-13T19:54:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741887_1065 (size=4783) 2024-11-13T19:54:15,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741887_1065 (size=4783) 2024-11-13T19:54:15,050 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf1/157880fb775746eba56e0f8460e4635d 2024-11-13T19:54:15,057 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf1/157880fb775746eba56e0f8460e4635d as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d 2024-11-13T19:54:15,064 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T19:54:15,065 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for ab2bd533f2e133bf9e32cd3965a8e16d in 41ms, sequenceid=5, compaction requested=false 2024-11-13T19:54:15,065 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T19:54:15,071 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T19:54:15,073 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,074 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for ab2bd533f2e133bf9e32cd3965a8e16d: Waiting for close lock at 1731527655024Running coprocessor pre-close hooks at 1731527655024Disabling compacts and flushes for region at 1731527655024Disabling writes for close at 1731527655024Obtaining lock to block concurrent updates at 1731527655024Preparing flush snapshotting stores in ab2bd533f2e133bf9e32cd3965a8e16d at 1731527655024Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731527655024Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. at 1731527655025 (+1 ms)Flushing ab2bd533f2e133bf9e32cd3965a8e16d/cf1: creating writer at 1731527655025Flushing ab2bd533f2e133bf9e32cd3965a8e16d/cf1: appending metadata at 1731527655041 (+16 ms)Flushing ab2bd533f2e133bf9e32cd3965a8e16d/cf1: closing flushed file at 1731527655042 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@366b1af8: reopening flushed file at 1731527655056 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for ab2bd533f2e133bf9e32cd3965a8e16d in 41ms, sequenceid=5, compaction requested=false at 1731527655065 (+9 ms)Writing region close event to WAL at 1731527655066 (+1 ms)Running coprocessor post-close hooks at 1731527655071 (+5 ms)Closed at 1731527655073 (+2 ms) 2024-11-13T19:54:15,074 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding ab2bd533f2e133bf9e32cd3965a8e16d move to 54d0784d1943,46741,1731527635545 record at close sequenceid=5 2024-11-13T19:54:15,077 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,078 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=CLOSED 2024-11-13T19:54:15,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668 because future has completed 2024-11-13T19:54:15,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T19:54:15,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668 in 217 msec 2024-11-13T19:54:15,086 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE; state=CLOSED, location=54d0784d1943,46741,1731527635545; forceNewPlan=false, retain=false 2024-11-13T19:54:15,131 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T19:54:15,131 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:15,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T19:54:15,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:15,237 INFO [54d0784d1943:35115 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T19:54:15,237 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPENING, regionLocation=54d0784d1943,46741,1731527635545 2024-11-13T19:54:15,240 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE because future has completed 2024-11-13T19:54:15,240 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545}] 2024-11-13T19:54:15,398 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,399 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ab2bd533f2e133bf9e32cd3965a8e16d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:15,399 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,399 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:15,399 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,399 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,401 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,402 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf1 2024-11-13T19:54:15,402 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:15,408 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d 2024-11-13T19:54:15,409 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:15,409 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,410 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf2 2024-11-13T19:54:15,410 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:15,410 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:15,411 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,411 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,412 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,413 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,413 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,414 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T19:54:15,416 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,417 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ab2bd533f2e133bf9e32cd3965a8e16d; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75045534, jitterRate=0.11826559901237488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T19:54:15,417 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,418 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ab2bd533f2e133bf9e32cd3965a8e16d: Running coprocessor pre-open hook at 1731527655399Writing region info on filesystem at 1731527655399Initializing all the Stores at 1731527655401 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527655401Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527655401Cleaning up temporary data from old regions at 1731527655413 (+12 ms)Running coprocessor post-open hooks at 1731527655417 (+4 ms)Region opened successfully at 1731527655418 (+1 ms) 2024-11-13T19:54:15,419 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., pid=9, masterSystemTime=1731527655393 2024-11-13T19:54:15,423 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,423 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPEN, openSeqNum=9, regionLocation=54d0784d1943,46741,1731527635545 2024-11-13T19:54:15,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545 because future has completed 2024-11-13T19:54:15,433 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=54d0784d1943,46741,1731527635545, table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T19:54:15,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T19:54:15,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545 in 194 msec 2024-11-13T19:54:15,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE in 584 msec 2024-11-13T19:54:15,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T19:54:15,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T19:54:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:44270 deadline: 1731527715469, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=46741 startCode=1731527635545. As of locationSeqNum=5. 2024-11-13T19:54:15,478 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=46741 startCode=1731527635545. As of locationSeqNum=5. 2024-11-13T19:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=46741 startCode=1731527635545. As of locationSeqNum=5. 2024-11-13T19:54:15,479 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,46741,1731527635545, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=46741 startCode=1731527635545. As of locationSeqNum=5. 2024-11-13T19:54:15,604 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab2bd533f2e133bf9e32cd3965a8e16d 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T19:54:15,628 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf1/693d0f2d63094d26abb6dcf31b9b2755 is 29, key is r1/cf1:/1731527655595/DeleteFamily/seqid=0 2024-11-13T19:54:15,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741888_1066 (size=4906) 2024-11-13T19:54:15,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741888_1066 (size=4906) 2024-11-13T19:54:15,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741888_1066 (size=4906) 2024-11-13T19:54:15,640 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf1/693d0f2d63094d26abb6dcf31b9b2755 2024-11-13T19:54:15,647 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 693d0f2d63094d26abb6dcf31b9b2755 2024-11-13T19:54:15,662 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf2/593aac359d324b42be35a7d4471d22bc is 29, key is r1/cf2:/1731527655595/DeleteFamily/seqid=0 2024-11-13T19:54:15,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741889_1067 (size=4906) 2024-11-13T19:54:15,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741889_1067 (size=4906) 2024-11-13T19:54:15,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741889_1067 (size=4906) 2024-11-13T19:54:15,671 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf2/593aac359d324b42be35a7d4471d22bc 2024-11-13T19:54:15,678 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 593aac359d324b42be35a7d4471d22bc 2024-11-13T19:54:15,679 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf1/693d0f2d63094d26abb6dcf31b9b2755 as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/693d0f2d63094d26abb6dcf31b9b2755 2024-11-13T19:54:15,685 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 693d0f2d63094d26abb6dcf31b9b2755 2024-11-13T19:54:15,686 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/693d0f2d63094d26abb6dcf31b9b2755, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T19:54:15,687 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf2/593aac359d324b42be35a7d4471d22bc as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/593aac359d324b42be35a7d4471d22bc 2024-11-13T19:54:15,694 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 593aac359d324b42be35a7d4471d22bc 2024-11-13T19:54:15,694 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/593aac359d324b42be35a7d4471d22bc, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T19:54:15,695 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for ab2bd533f2e133bf9e32cd3965a8e16d in 91ms, sequenceid=12, compaction requested=false 2024-11-13T19:54:15,695 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ab2bd533f2e133bf9e32cd3965a8e16d: 2024-11-13T19:54:15,699 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T19:54:15,700 DEBUG [Time-limited test {}] regionserver.HStore(1541): ab2bd533f2e133bf9e32cd3965a8e16d/cf1 is initiating major compaction (all files) 2024-11-13T19:54:15,701 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T19:54:15,701 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:54:15,701 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ab2bd533f2e133bf9e32cd3965a8e16d/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,702 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d, hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/693d0f2d63094d26abb6dcf31b9b2755] into tmpdir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp, totalSize=9.5 K 2024-11-13T19:54:15,703 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 157880fb775746eba56e0f8460e4635d, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731527654836 2024-11-13T19:54:15,703 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 693d0f2d63094d26abb6dcf31b9b2755, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T19:54:15,716 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ab2bd533f2e133bf9e32cd3965a8e16d#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:54:15,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741890_1068 (size=4626) 2024-11-13T19:54:15,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741890_1068 (size=4626) 2024-11-13T19:54:15,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741890_1068 (size=4626) 2024-11-13T19:54:15,731 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf1/52e13ee0ba16447c84f8a32c18bb52d5 as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/52e13ee0ba16447c84f8a32c18bb52d5 2024-11-13T19:54:15,745 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in ab2bd533f2e133bf9e32cd3965a8e16d/cf1 of ab2bd533f2e133bf9e32cd3965a8e16d into 52e13ee0ba16447c84f8a32c18bb52d5(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:54:15,745 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ab2bd533f2e133bf9e32cd3965a8e16d: 2024-11-13T19:54:15,745 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T19:54:15,745 DEBUG [Time-limited test {}] regionserver.HStore(1541): ab2bd533f2e133bf9e32cd3965a8e16d/cf2 is initiating major compaction (all files) 2024-11-13T19:54:15,745 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T19:54:15,745 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:54:15,745 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of ab2bd533f2e133bf9e32cd3965a8e16d/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,745 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/593aac359d324b42be35a7d4471d22bc] into tmpdir=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp, totalSize=4.8 K 2024-11-13T19:54:15,746 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 593aac359d324b42be35a7d4471d22bc, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T19:54:15,751 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): ab2bd533f2e133bf9e32cd3965a8e16d#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:54:15,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741891_1069 (size=4592) 2024-11-13T19:54:15,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741891_1069 (size=4592) 2024-11-13T19:54:15,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741891_1069 (size=4592) 2024-11-13T19:54:15,765 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/.tmp/cf2/5b9768e437d14a1fb7989a370eca00a0 as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/5b9768e437d14a1fb7989a370eca00a0 2024-11-13T19:54:15,772 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in ab2bd533f2e133bf9e32cd3965a8e16d/cf2 of ab2bd533f2e133bf9e32cd3965a8e16d into 5b9768e437d14a1fb7989a370eca00a0(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:54:15,772 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for ab2bd533f2e133bf9e32cd3965a8e16d: 2024-11-13T19:54:15,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ab2bd533f2e133bf9e32cd3965a8e16d, source=54d0784d1943,46741,1731527635545, destination=54d0784d1943,36813,1731527635668, warming up region on 54d0784d1943,36813,1731527635668 2024-11-13T19:54:15,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ab2bd533f2e133bf9e32cd3965a8e16d, source=54d0784d1943,46741,1731527635545, destination=54d0784d1943,36813,1731527635668, running balancer 2024-11-13T19:54:15,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE 2024-11-13T19:54:15,779 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE 2024-11-13T19:54:15,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(7855): Warmup {ENCODED => ab2bd533f2e133bf9e32cd3965a8e16d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:15,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:15,781 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=CLOSING, regionLocation=54d0784d1943,46741,1731527635545 2024-11-13T19:54:15,782 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,783 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf1 2024-11-13T19:54:15,783 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:15,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE because future has completed 2024-11-13T19:54:15,786 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:54:15,786 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545}] 2024-11-13T19:54:15,796 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d 2024-11-13T19:54:15,801 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/52e13ee0ba16447c84f8a32c18bb52d5 2024-11-13T19:54:15,805 INFO [StoreFileOpener-ab2bd533f2e133bf9e32cd3965a8e16d-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 693d0f2d63094d26abb6dcf31b9b2755 2024-11-13T19:54:15,806 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/693d0f2d63094d26abb6dcf31b9b2755 2024-11-13T19:54:15,806 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:15,806 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,807 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf2 2024-11-13T19:54:15,807 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:15,814 INFO [StoreFileOpener-ab2bd533f2e133bf9e32cd3965a8e16d-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 593aac359d324b42be35a7d4471d22bc 2024-11-13T19:54:15,814 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/593aac359d324b42be35a7d4471d22bc 2024-11-13T19:54:15,821 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/5b9768e437d14a1fb7989a370eca00a0 2024-11-13T19:54:15,821 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(1722): Closing ab2bd533f2e133bf9e32cd3965a8e16d, disabling compactions & flushes 2024-11-13T19:54:15,822 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. after waiting 0 ms 2024-11-13T19:54:15,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,824 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] regionserver.HRegion(1676): Region close journal for ab2bd533f2e133bf9e32cd3965a8e16d: Waiting for close lock at 1731527655822Disabling compacts and flushes for region at 1731527655822Disabling writes for close at 1731527655822Writing region close event to WAL at 1731527655824 (+2 ms)Closed at 1731527655824 2024-11-13T19:54:15,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T19:54:15,940 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,941 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:54:15,941 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing ab2bd533f2e133bf9e32cd3965a8e16d, disabling compactions & flushes 2024-11-13T19:54:15,941 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,941 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,941 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. after waiting 0 ms 2024-11-13T19:54:15,941 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,942 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d, hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/693d0f2d63094d26abb6dcf31b9b2755] to archive 2024-11-13T19:54:15,946 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:54:15,950 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d to hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/157880fb775746eba56e0f8460e4635d 2024-11-13T19:54:15,953 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/693d0f2d63094d26abb6dcf31b9b2755 to hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/693d0f2d63094d26abb6dcf31b9b2755 2024-11-13T19:54:15,971 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/593aac359d324b42be35a7d4471d22bc] to archive 2024-11-13T19:54:15,972 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:54:15,975 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/593aac359d324b42be35a7d4471d22bc to hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/593aac359d324b42be35a7d4471d22bc 2024-11-13T19:54:15,981 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T19:54:15,981 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:15,981 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for ab2bd533f2e133bf9e32cd3965a8e16d: Waiting for close lock at 1731527655941Running coprocessor pre-close hooks at 1731527655941Disabling compacts and flushes for region at 1731527655941Disabling writes for close at 1731527655941Writing region close event to WAL at 1731527655976 (+35 ms)Running coprocessor post-close hooks at 1731527655981 (+5 ms)Closed at 1731527655981 2024-11-13T19:54:15,982 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding ab2bd533f2e133bf9e32cd3965a8e16d move to 54d0784d1943,36813,1731527635668 record at close sequenceid=12 2024-11-13T19:54:15,985 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:15,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=CLOSED 2024-11-13T19:54:15,988 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545 because future has completed 2024-11-13T19:54:15,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T19:54:15,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545 in 206 msec 2024-11-13T19:54:15,996 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE; state=CLOSED, location=54d0784d1943,36813,1731527635668; forceNewPlan=false, retain=false 2024-11-13T19:54:16,147 INFO [54d0784d1943:35115 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T19:54:16,147 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPENING, regionLocation=54d0784d1943,36813,1731527635668 2024-11-13T19:54:16,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE because future has completed 2024-11-13T19:54:16,151 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668}] 2024-11-13T19:54:16,310 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,310 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => ab2bd533f2e133bf9e32cd3965a8e16d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:16,310 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,310 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:16,311 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,311 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,312 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,313 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf1 2024-11-13T19:54:16,313 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:16,320 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/52e13ee0ba16447c84f8a32c18bb52d5 2024-11-13T19:54:16,320 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:16,320 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,321 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf2 2024-11-13T19:54:16,321 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:16,327 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/5b9768e437d14a1fb7989a370eca00a0 2024-11-13T19:54:16,328 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:16,328 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,329 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,330 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,331 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,331 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,332 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T19:54:16,333 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,334 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened ab2bd533f2e133bf9e32cd3965a8e16d; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71807699, jitterRate=0.07001809775829315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T19:54:16,334 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,335 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for ab2bd533f2e133bf9e32cd3965a8e16d: Running coprocessor pre-open hook at 1731527656311Writing region info on filesystem at 1731527656311Initializing all the Stores at 1731527656312 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527656312Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527656312Cleaning up temporary data from old regions at 1731527656331 (+19 ms)Running coprocessor post-open hooks at 1731527656334 (+3 ms)Region opened successfully at 1731527656335 (+1 ms) 2024-11-13T19:54:16,336 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., pid=12, masterSystemTime=1731527656305 2024-11-13T19:54:16,340 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,340 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,341 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPEN, openSeqNum=18, regionLocation=54d0784d1943,36813,1731527635668 2024-11-13T19:54:16,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668 because future has completed 2024-11-13T19:54:16,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T19:54:16,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,36813,1731527635668 in 194 msec 2024-11-13T19:54:16,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, REOPEN/MOVE in 571 msec 2024-11-13T19:54:16,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T19:54:16,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T19:54:16,385 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 54d0784d1943,36813,1731527635668: testing ***** 2024-11-13T19:54:16,385 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T19:54:16,387 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T19:54:16,389 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T19:54:16,394 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T19:54:16,396 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T19:54:16,409 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 483656480 }, "NonHeapMemoryUsage": { "committed": 172752896, "init": 7667712, "max": -1, "used": 170038816 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "54d0784d1943", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 1, "ProcessCallTime_max": 47, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 4, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 17, "ProcessCallTime_98th_percentile": 47, "ProcessCallTime_99th_percentile": 47, "ProcessCallTime_99.9th_percentile": 47, "ProcessCallTime_TimeRangeCount_0-1": 13, "ProcessCallTime_TimeRangeCount_1-3": 10, "ProcessCallTime_TimeRangeCount_3-10": 6, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 1, "TotalCallTime_max": 47, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 17, "TotalCallTime_98th_percentile": 47, "TotalCallTime_99th_percentile": 47, "TotalCallTime_99.9th_percentile": 47, "TotalCallTime_TimeRangeCount_0-1": 13, "TotalCallTime_TimeRangeCount_1-3": 9, "TotalCallTime_TimeRangeCount_3-10": 7, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 160, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5782 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "54d0784d1943", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57594", "tag.serverName": "54d0784d1943,35943,1731527635712", "tag.clusterId": "697cb3e1-bdea-4df8-92f0-0da5980bf2ec", "tag.Context": "regionserver", "tag.Hostname": "54d0784d1943", "regionCount": 0, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731527635712, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 1.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T19:54:16,413 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35115 {}] master.MasterRpcServices(700): 54d0784d1943,36813,1731527635668 reported a fatal error: ***** ABORTING region server 54d0784d1943,36813,1731527635668: testing ***** 2024-11-13T19:54:16,416 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '54d0784d1943,36813,1731527635668' ***** 2024-11-13T19:54:16,416 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T19:54:16,416 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T19:54:16,416 INFO [RS:1;54d0784d1943:36813 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T19:54:16,417 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T19:54:16,417 INFO [RS:1;54d0784d1943:36813 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T19:54:16,417 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(3091): Received CLOSE for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:16,417 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(956): aborting server 54d0784d1943,36813,1731527635668 2024-11-13T19:54:16,417 INFO [RS:1;54d0784d1943:36813 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:54:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46741 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:50384 deadline: 1731527716417, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=36813 startCode=1731527635668. As of locationSeqNum=12. 2024-11-13T19:54:16,418 INFO [RS:1;54d0784d1943:36813 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;54d0784d1943:36813. 2024-11-13T19:54:16,418 DEBUG [RS:1;54d0784d1943:36813 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:54:16,418 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab2bd533f2e133bf9e32cd3965a8e16d, disabling compactions & flushes 2024-11-13T19:54:16,418 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,418 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,418 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. after waiting 0 ms 2024-11-13T19:54:16,418 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,46741,1731527635545, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,46741,1731527635545, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=36813 startCode=1731527635668. As of locationSeqNum=12. 2024-11-13T19:54:16,418 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,418 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,46741,1731527635545, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=36813 startCode=1731527635668. As of locationSeqNum=12. 2024-11-13T19:54:16,418 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,46741,1731527635545, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=54d0784d1943 port=36813 startCode=1731527635668. As of locationSeqNum=12. 2024-11-13T19:54:16,418 DEBUG [RS:1;54d0784d1943:36813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:16,420 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,420 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T19:54:16,420 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab2bd533f2e133bf9e32cd3965a8e16d: Waiting for close lock at 1731527656418Running coprocessor pre-close hooks at 1731527656418Disabling compacts and flushes for region at 1731527656418Disabling writes for close at 1731527656418Writing region close event to WAL at 1731527656419 (+1 ms)Running coprocessor post-close hooks at 1731527656419Closed at 1731527656420 (+1 ms) 2024-11-13T19:54:16,420 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1325): Online Regions={ab2bd533f2e133bf9e32cd3965a8e16d=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.} 2024-11-13T19:54:16,420 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:16,420 DEBUG [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1351): Waiting on 2024-11-13T19:54:16,479 INFO [regionserver/54d0784d1943:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:54:16,524 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 54d0784d1943,36813,1731527635668 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:16,525 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 54d0784d1943,36813,1731527635668 aborting 2024-11-13T19:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 54d0784d1943,36813,1731527635668 aborting 2024-11-13T19:54:16,526 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=12 from cache 2024-11-13T19:54:16,621 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(976): stopping server 54d0784d1943,36813,1731527635668; all regions closed. 2024-11-13T19:54:16,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741834_1010 (size=2288) 2024-11-13T19:54:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741834_1010 (size=2288) 2024-11-13T19:54:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741834_1010 (size=2288) 2024-11-13T19:54:16,626 DEBUG [RS:1;54d0784d1943:36813 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:16,626 INFO [RS:1;54d0784d1943:36813 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:54:16,626 INFO [RS:1;54d0784d1943:36813 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:54:16,627 INFO [RS:1;54d0784d1943:36813 {}] hbase.ChoreService(370): Chore service for: regionserver/54d0784d1943:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T19:54:16,627 INFO [regionserver/54d0784d1943:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:54:16,627 INFO [RS:1;54d0784d1943:36813 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T19:54:16,627 INFO [RS:1;54d0784d1943:36813 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T19:54:16,627 INFO [RS:1;54d0784d1943:36813 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T19:54:16,627 INFO [RS:1;54d0784d1943:36813 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:54:16,627 INFO [RS:1;54d0784d1943:36813 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36813 2024-11-13T19:54:16,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/54d0784d1943,36813,1731527635668 2024-11-13T19:54:16,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:54:16,636 INFO [RS:1;54d0784d1943:36813 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:54:16,637 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [54d0784d1943,36813,1731527635668] 2024-11-13T19:54:16,639 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/54d0784d1943,36813,1731527635668 already deleted, retry=false 2024-11-13T19:54:16,639 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 54d0784d1943,36813,1731527635668 on 54d0784d1943,35115,1731527634648 2024-11-13T19:54:16,646 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 54d0784d1943,36813,1731527635668, splitWal=true, meta=false 2024-11-13T19:54:16,649 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 54d0784d1943,36813,1731527635668 (carryingMeta=false) 54d0784d1943,36813,1731527635668/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@76e04042[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T19:54:16,649 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 54d0784d1943,36813,1731527635668, splitWal=true, meta=false 2024-11-13T19:54:16,651 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 54d0784d1943,36813,1731527635668 had 1 regions 2024-11-13T19:54:16,652 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 54d0784d1943,36813,1731527635668, splitWal=true, meta=false, isMeta: false 2024-11-13T19:54:16,654 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting 2024-11-13T19:54:16,655 INFO [PEWorker-2 {}] master.SplitWALManager(105): 54d0784d1943,36813,1731527635668 WAL count=1, meta=false 2024-11-13T19:54:16,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 54d0784d1943%2C36813%2C1731527635668.1731527637542}] 2024-11-13T19:54:16,665 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=54d0784d1943,46741,1731527635545 2024-11-13T19:54:16,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 54d0784d1943%2C36813%2C1731527635668.1731527637542, worker=54d0784d1943,46741,1731527635545}] 2024-11-13T19:54:16,738 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18] 2024-11-13T19:54:16,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:16,739 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36813-0x100eda9e0200002, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:16,740 INFO [RS:1;54d0784d1943:36813 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:54:16,740 INFO [RS:1;54d0784d1943:36813 {}] regionserver.HRegionServer(1031): Exiting; stopping=54d0784d1943,36813,1731527635668; zookeeper connection closed. 2024-11-13T19:54:16,740 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 54d0784d1943:36813 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 54d0784d1943/172.17.0.2:36813 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:16,740 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1b804d35 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1b804d35 2024-11-13T19:54:16,741 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18, error=java.net.ConnectException: Call to address=54d0784d1943:36813 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 54d0784d1943/172.17.0.2:36813 2024-11-13T19:54:16,741 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-13T19:54:16,741 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18 from cache 2024-11-13T19:54:16,741 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 54d0784d1943:36813 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 54d0784d1943/172.17.0.2:36813 2024-11-13T19:54:16,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46741 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T19:54:16,852 INFO [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542, size=2.2 K (2288bytes) 2024-11-13T19:54:16,852 INFO [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542 2024-11-13T19:54:16,853 INFO [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542 after 0ms 2024-11-13T19:54:16,886 DEBUG [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:16,886 INFO [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542 took 34ms 2024-11-13T19:54:16,898 DEBUG [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for ab2bd533f2e133bf9e32cd3965a8e16d: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T19:54:16,898 DEBUG [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542 so closing down 2024-11-13T19:54:16,899 DEBUG [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:16,899 INFO [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:16,899 INFO [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-13T19:54:16,899 DEBUG [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542, journal: Splitting hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542, size=2.2 K (2288bytes) at 1731527656852Finishing writing output for hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542 so closing down at 1731527656898 (+46 ms)3 split writer threads finished at 1731527656899 (+1 ms)Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1731527656899 2024-11-13T19:54:16,899 DEBUG [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542 2024-11-13T19:54:16,901 DEBUG [RS_LOG_REPLAY_OPS-regionserver/54d0784d1943:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T19:54:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35115 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T19:54:16,908 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting/54d0784d1943%2C36813%2C1731527635668.1731527637542 to hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs 2024-11-13T19:54:16,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T19:54:16,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 54d0784d1943%2C36813%2C1731527635668.1731527637542, worker=54d0784d1943,46741,1731527635545 in 241 msec 2024-11-13T19:54:16,914 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=54d0784d1943,46741,1731527635545 2024-11-13T19:54:16,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T19:54:16,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 54d0784d1943%2C36813%2C1731527635668.1731527637542, worker=54d0784d1943,46741,1731527635545 in 257 msec 2024-11-13T19:54:16,919 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting dir is empty, no logs to split. 2024-11-13T19:54:16,919 INFO [PEWorker-2 {}] master.SplitWALManager(105): 54d0784d1943,36813,1731527635668 WAL count=0, meta=false 2024-11-13T19:54:16,919 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 54d0784d1943,36813,1731527635668 WAL splitting is done? wals=0, meta=false 2024-11-13T19:54:16,921 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 54d0784d1943,36813,1731527635668 failed, ignore...File hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/WALs/54d0784d1943,36813,1731527635668-splitting does not exist. 2024-11-13T19:54:16,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN}] 2024-11-13T19:54:16,924 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN 2024-11-13T19:54:16,926 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T19:54:17,047 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18] 2024-11-13T19:54:17,048 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 54d0784d1943:36813 this server is in the failed servers list 2024-11-13T19:54:17,049 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=54d0784d1943:36813 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 54d0784d1943:36813 2024-11-13T19:54:17,049 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 54d0784d1943:36813 2024-11-13T19:54:17,049 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,36813,1731527635668, seqNum=18 from cache 2024-11-13T19:54:17,076 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(204): Hosts are {54d0784d1943=0} racks are {/default-rack=0} 2024-11-13T19:54:17,076 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T19:54:17,076 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T19:54:17,076 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T19:54:17,076 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T19:54:17,076 INFO [54d0784d1943:35115 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T19:54:17,076 INFO [54d0784d1943:35115 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T19:54:17,076 DEBUG [54d0784d1943:35115 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T19:54:17,077 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPENING, regionLocation=54d0784d1943,46741,1731527635545 2024-11-13T19:54:17,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN because future has completed 2024-11-13T19:54:17,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545}] 2024-11-13T19:54:17,238 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:17,238 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => ab2bd533f2e133bf9e32cd3965a8e16d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:17,238 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,238 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:17,238 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,238 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,240 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,241 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf1 2024-11-13T19:54:17,241 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,248 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf1/52e13ee0ba16447c84f8a32c18bb52d5 2024-11-13T19:54:17,248 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,248 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,249 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab2bd533f2e133bf9e32cd3965a8e16d columnFamilyName cf2 2024-11-13T19:54:17,249 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,256 DEBUG [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/cf2/5b9768e437d14a1fb7989a370eca00a0 2024-11-13T19:54:17,256 INFO [StoreOpener-ab2bd533f2e133bf9e32cd3965a8e16d-1 {}] regionserver.HStore(327): Store=ab2bd533f2e133bf9e32cd3965a8e16d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,256 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,257 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,259 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,260 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,260 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,261 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T19:54:17,262 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,263 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened ab2bd533f2e133bf9e32cd3965a8e16d; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74963779, jitterRate=0.11704735457897186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T19:54:17,263 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:17,264 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for ab2bd533f2e133bf9e32cd3965a8e16d: Running coprocessor pre-open hook at 1731527657239Writing region info on filesystem at 1731527657239Initializing all the Stores at 1731527657240 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657240Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657240Cleaning up temporary data from old regions at 1731527657260 (+20 ms)Running coprocessor post-open hooks at 1731527657263 (+3 ms)Region opened successfully at 1731527657264 (+1 ms) 2024-11-13T19:54:17,266 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., pid=17, masterSystemTime=1731527657233 2024-11-13T19:54:17,268 DEBUG [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:17,268 INFO [RS_OPEN_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:17,270 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ab2bd533f2e133bf9e32cd3965a8e16d, regionState=OPEN, openSeqNum=18, regionLocation=54d0784d1943,46741,1731527635545 2024-11-13T19:54:17,271 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35115 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=54d0784d1943,46741,1731527635545, table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T19:54:17,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545 because future has completed 2024-11-13T19:54:17,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T19:54:17,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure ab2bd533f2e133bf9e32cd3965a8e16d, server=54d0784d1943,46741,1731527635545 in 194 msec 2024-11-13T19:54:17,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T19:54:17,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=ab2bd533f2e133bf9e32cd3965a8e16d, ASSIGN in 354 msec 2024-11-13T19:54:17,281 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 54d0784d1943,36813,1731527635668 after splitting done 2024-11-13T19:54:17,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 54d0784d1943,36813,1731527635668, splitWal=true, meta=false in 641 msec 2024-11-13T19:54:17,558 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d., hostname=54d0784d1943,46741,1731527635545, seqNum=18] 2024-11-13T19:54:17,580 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=401 (was 410), OpenFileDescriptor=1042 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=529 (was 529), ProcessCount=11 (was 11), AvailableMemoryMB=2413 (was 2496) 2024-11-13T19:54:17,583 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-13T19:54:17,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=399, OpenFileDescriptor=1042, MaxFileDescriptor=1048576, SystemLoadAverage=529, ProcessCount=11, AvailableMemoryMB=2411 2024-11-13T19:54:17,601 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1042 is superior to 1024 2024-11-13T19:54:17,626 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:17,631 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:17,632 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:17,635 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-34446513, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-34446513, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:17,655 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-34446513/hregion-34446513.1731527657636, exclude list is [], retry=0 2024-11-13T19:54:17,659 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:17,659 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:17,659 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:17,689 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-34446513/hregion-34446513.1731527657636 2024-11-13T19:54:17,689 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:17,689 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8beb8990cf896be74f5dd943a008c700, NAME => 'testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:17,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741893_1071 (size=67) 2024-11-13T19:54:17,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741893_1071 (size=67) 2024-11-13T19:54:17,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741893_1071 (size=67) 2024-11-13T19:54:17,702 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:17,703 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,704 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName a 2024-11-13T19:54:17,704 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,705 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,705 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,706 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName b 2024-11-13T19:54:17,707 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,707 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,707 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,708 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName c 2024-11-13T19:54:17,709 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,709 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,709 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,710 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,711 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,712 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,712 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,712 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:17,714 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,716 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:17,716 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8beb8990cf896be74f5dd943a008c700; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68902917, jitterRate=0.02673347294330597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:17,717 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8beb8990cf896be74f5dd943a008c700: Writing region info on filesystem at 1731527657702Initializing all the Stores at 1731527657703 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657703Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657703Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657703Cleaning up temporary data from old regions at 1731527657712 (+9 ms)Region opened successfully at 1731527657717 (+5 ms) 2024-11-13T19:54:17,717 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8beb8990cf896be74f5dd943a008c700, disabling compactions & flushes 2024-11-13T19:54:17,717 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:17,717 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:17,717 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. after waiting 0 ms 2024-11-13T19:54:17,717 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:17,718 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:17,718 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8beb8990cf896be74f5dd943a008c700: Waiting for close lock at 1731527657717Disabling compacts and flushes for region at 1731527657717Disabling writes for close at 1731527657717Writing region close event to WAL at 1731527657718 (+1 ms)Closed at 1731527657718 2024-11-13T19:54:17,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741892_1070 (size=175) 2024-11-13T19:54:17,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741892_1070 (size=175) 2024-11-13T19:54:17,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741892_1070 (size=175) 2024-11-13T19:54:17,723 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:17,723 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-34446513:(num 1731527657636) 2024-11-13T19:54:17,723 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:17,725 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:17,745 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726, exclude list is [], retry=0 2024-11-13T19:54:17,749 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:17,750 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:17,750 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:17,778 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726 2024-11-13T19:54:17,779 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:17,779 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8beb8990cf896be74f5dd943a008c700, NAME => 'testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:17,780 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:17,780 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,780 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,781 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,782 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName a 2024-11-13T19:54:17,782 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,783 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,783 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,784 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName b 2024-11-13T19:54:17,784 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,785 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,785 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,786 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName c 2024-11-13T19:54:17,786 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:17,786 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:17,786 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,787 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,788 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,789 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,789 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,790 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:17,791 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:17,792 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8beb8990cf896be74f5dd943a008c700; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63533139, jitterRate=-0.05328245460987091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:17,792 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8beb8990cf896be74f5dd943a008c700: Writing region info on filesystem at 1731527657780Initializing all the Stores at 1731527657781 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657781Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657781Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527657781Cleaning up temporary data from old regions at 1731527657789 (+8 ms)Region opened successfully at 1731527657792 (+3 ms) 2024-11-13T19:54:17,824 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8beb8990cf896be74f5dd943a008c700 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T19:54:17,848 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/a/0420d37e82804533a52c0e3c0df11e7d is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731527657792/Put/seqid=0 2024-11-13T19:54:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741895_1073 (size=5958) 2024-11-13T19:54:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741895_1073 (size=5958) 2024-11-13T19:54:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741895_1073 (size=5958) 2024-11-13T19:54:17,867 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/a/0420d37e82804533a52c0e3c0df11e7d 2024-11-13T19:54:17,897 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/b/47104d4703f7438b956046db89b53c83 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731527657803/Put/seqid=0 2024-11-13T19:54:17,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741896_1074 (size=5958) 2024-11-13T19:54:17,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741896_1074 (size=5958) 2024-11-13T19:54:17,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741896_1074 (size=5958) 2024-11-13T19:54:18,309 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/b/47104d4703f7438b956046db89b53c83 2024-11-13T19:54:18,339 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/c/c3697d388cb046f9bed7c0886ab5a352 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731527657813/Put/seqid=0 2024-11-13T19:54:18,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741897_1075 (size=5958) 2024-11-13T19:54:18,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741897_1075 (size=5958) 2024-11-13T19:54:18,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741897_1075 (size=5958) 2024-11-13T19:54:18,349 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/c/c3697d388cb046f9bed7c0886ab5a352 2024-11-13T19:54:18,356 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/a/0420d37e82804533a52c0e3c0df11e7d as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/a/0420d37e82804533a52c0e3c0df11e7d 2024-11-13T19:54:18,362 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/a/0420d37e82804533a52c0e3c0df11e7d, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T19:54:18,363 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/b/47104d4703f7438b956046db89b53c83 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/b/47104d4703f7438b956046db89b53c83 2024-11-13T19:54:18,368 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/b/47104d4703f7438b956046db89b53c83, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T19:54:18,369 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/c/c3697d388cb046f9bed7c0886ab5a352 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/c/c3697d388cb046f9bed7c0886ab5a352 2024-11-13T19:54:18,379 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/c/c3697d388cb046f9bed7c0886ab5a352, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T19:54:18,382 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 8beb8990cf896be74f5dd943a008c700 in 558ms, sequenceid=33, compaction requested=false 2024-11-13T19:54:18,382 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8beb8990cf896be74f5dd943a008c700: 2024-11-13T19:54:18,382 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8beb8990cf896be74f5dd943a008c700, disabling compactions & flushes 2024-11-13T19:54:18,382 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:18,382 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:18,382 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. after waiting 0 ms 2024-11-13T19:54:18,382 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:18,385 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. 2024-11-13T19:54:18,385 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8beb8990cf896be74f5dd943a008c700: Waiting for close lock at 1731527658382Disabling compacts and flushes for region at 1731527658382Disabling writes for close at 1731527658382Writing region close event to WAL at 1731527658385 (+3 ms)Closed at 1731527658385 2024-11-13T19:54:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741894_1072 (size=6311) 2024-11-13T19:54:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741894_1072 (size=6311) 2024-11-13T19:54:18,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741894_1072 (size=6311) 2024-11-13T19:54:18,395 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/b/47104d4703f7438b956046db89b53c83 to hdfs://localhost:39891/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/b/47104d4703f7438b956046db89b53c83 2024-11-13T19:54:18,412 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726, size=6.2 K (6311bytes) 2024-11-13T19:54:18,412 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726 2024-11-13T19:54:18,413 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726 after 1ms 2024-11-13T19:54:18,449 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:18,449 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726 took 37ms 2024-11-13T19:54:18,461 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726 so closing down 2024-11-13T19:54:18,462 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:18,464 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731527657726.temp 2024-11-13T19:54:18,508 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000003-wal.1731527657726.temp 2024-11-13T19:54:18,508 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741898_1076 (size=5575) 2024-11-13T19:54:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741898_1076 (size=5575) 2024-11-13T19:54:18,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741898_1076 (size=5575) 2024-11-13T19:54:18,918 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000003-wal.1731527657726.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:18,920 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000003-wal.1731527657726.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000032 2024-11-13T19:54:18,920 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 467 ms; skipped=2; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-13T19:54:18,920 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726, journal: Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726, size=6.2 K (6311bytes) at 1731527658412Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726 so closing down at 1731527658461 (+49 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000003-wal.1731527657726.temp at 1731527658508 (+47 ms)3 split writer threads finished at 1731527658508Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000003-wal.1731527657726.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731527658918 (+410 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000003-wal.1731527657726.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000032 at 1731527658920 (+2 ms)Processed 32 edits across 1 Regions in 467 ms; skipped=2; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1731527658920 2024-11-13T19:54:18,922 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527657726 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527657726 2024-11-13T19:54:18,923 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000032 2024-11-13T19:54:18,923 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:18,925 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:18,941 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527658925, exclude list is [], retry=0 2024-11-13T19:54:18,945 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:18,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:18,946 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:18,997 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527658925 2024-11-13T19:54:19,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:19,001 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8beb8990cf896be74f5dd943a008c700, NAME => 'testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:19,002 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:19,002 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,002 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,004 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,005 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName a 2024-11-13T19:54:19,005 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:19,012 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/a/0420d37e82804533a52c0e3c0df11e7d 2024-11-13T19:54:19,012 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:19,012 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,014 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName b 2024-11-13T19:54:19,014 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:19,014 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:19,014 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,015 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8beb8990cf896be74f5dd943a008c700 columnFamilyName c 2024-11-13T19:54:19,015 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:19,030 DEBUG [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/c/c3697d388cb046f9bed7c0886ab5a352 2024-11-13T19:54:19,030 INFO [StoreOpener-8beb8990cf896be74f5dd943a008c700-1 {}] regionserver.HStore(327): Store=8beb8990cf896be74f5dd943a008c700/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:19,031 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,031 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,034 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,035 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000032 2024-11-13T19:54:19,079 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:19,082 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000032 2024-11-13T19:54:19,082 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8beb8990cf896be74f5dd943a008c700 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T19:54:19,122 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/b/f5cac9e920374e7ba7877004e31957cb is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731527657803/Put/seqid=0 2024-11-13T19:54:19,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741900_1078 (size=5958) 2024-11-13T19:54:19,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741900_1078 (size=5958) 2024-11-13T19:54:19,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741900_1078 (size=5958) 2024-11-13T19:54:19,157 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/b/f5cac9e920374e7ba7877004e31957cb 2024-11-13T19:54:19,186 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/.tmp/b/f5cac9e920374e7ba7877004e31957cb as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/b/f5cac9e920374e7ba7877004e31957cb 2024-11-13T19:54:19,198 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/b/f5cac9e920374e7ba7877004e31957cb, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T19:54:19,198 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 8beb8990cf896be74f5dd943a008c700 in 116ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T19:54:19,199 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/0000000000000000032 2024-11-13T19:54:19,202 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,202 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,203 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:19,206 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8beb8990cf896be74f5dd943a008c700 2024-11-13T19:54:19,210 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/8beb8990cf896be74f5dd943a008c700/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T19:54:19,213 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8beb8990cf896be74f5dd943a008c700; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62099150, jitterRate=-0.07465055584907532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:19,215 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8beb8990cf896be74f5dd943a008c700: Writing region info on filesystem at 1731527659002Initializing all the Stores at 1731527659003 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527659003Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527659004 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527659004Obtaining lock to block concurrent updates at 1731527659082 (+78 ms)Preparing flush snapshotting stores in 8beb8990cf896be74f5dd943a008c700 at 1731527659082Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731527659082Flushing stores of testReplayEditsWrittenViaHRegion,,1731527657626.8beb8990cf896be74f5dd943a008c700. at 1731527659082Flushing 8beb8990cf896be74f5dd943a008c700/b: creating writer at 1731527659082Flushing 8beb8990cf896be74f5dd943a008c700/b: appending metadata at 1731527659121 (+39 ms)Flushing 8beb8990cf896be74f5dd943a008c700/b: closing flushed file at 1731527659121Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ab0c3f: reopening flushed file at 1731527659184 (+63 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 8beb8990cf896be74f5dd943a008c700 in 116ms, sequenceid=32, compaction requested=false; wal=null at 1731527659198 (+14 ms)Cleaning up temporary data from old regions at 1731527659202 (+4 ms)Region opened successfully at 1731527659215 (+13 ms) 2024-11-13T19:54:19,250 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=407 (was 399) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:55692 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:42310 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:41808 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:42462 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:55562 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:41940 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1112 (was 1042) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=503 (was 529), ProcessCount=11 (was 11), AvailableMemoryMB=2285 (was 2411) 2024-11-13T19:54:19,250 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1112 is superior to 1024 2024-11-13T19:54:19,272 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=407, OpenFileDescriptor=1112, MaxFileDescriptor=1048576, SystemLoadAverage=503, ProcessCount=11, AvailableMemoryMB=2281 2024-11-13T19:54:19,272 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1112 is superior to 1024 2024-11-13T19:54:19,299 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:19,303 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:19,304 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:19,312 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30127697, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-30127697, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:19,329 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30127697/hregion-30127697.1731527659312, exclude list is [], retry=0 2024-11-13T19:54:19,337 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:19,340 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:19,350 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:19,405 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30127697/hregion-30127697.1731527659312 2024-11-13T19:54:19,410 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:19,410 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 394d753144b126e9496d21c901cb380c, NAME => 'testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741902_1080 (size=68) 2024-11-13T19:54:19,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741902_1080 (size=68) 2024-11-13T19:54:19,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741902_1080 (size=68) 2024-11-13T19:54:19,856 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:19,857 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,859 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName a 2024-11-13T19:54:19,859 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:19,860 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:19,860 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,861 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName b 2024-11-13T19:54:19,861 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:19,862 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:19,862 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,864 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName c 2024-11-13T19:54:19,864 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:19,864 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:19,864 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,865 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,866 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,867 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,867 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,868 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:19,869 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:19,872 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:19,872 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 394d753144b126e9496d21c901cb380c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72785869, jitterRate=0.08459396660327911}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:19,873 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 394d753144b126e9496d21c901cb380c: Writing region info on filesystem at 1731527659856Initializing all the Stores at 1731527659857 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527659857Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527659857Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527659857Cleaning up temporary data from old regions at 1731527659867 (+10 ms)Region opened successfully at 1731527659873 (+6 ms) 2024-11-13T19:54:19,873 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 394d753144b126e9496d21c901cb380c, disabling compactions & flushes 2024-11-13T19:54:19,873 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:19,873 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:19,873 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. after waiting 0 ms 2024-11-13T19:54:19,873 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:19,877 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:19,877 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 394d753144b126e9496d21c901cb380c: Waiting for close lock at 1731527659873Disabling compacts and flushes for region at 1731527659873Disabling writes for close at 1731527659873Writing region close event to WAL at 1731527659877 (+4 ms)Closed at 1731527659877 2024-11-13T19:54:19,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741901_1079 (size=175) 2024-11-13T19:54:19,881 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-30127697/hregion-30127697.1731527659312 not finished, retry = 0 2024-11-13T19:54:19,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741901_1079 (size=175) 2024-11-13T19:54:19,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741901_1079 (size=175) 2024-11-13T19:54:19,984 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:19,984 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30127697:(num 1731527659312) 2024-11-13T19:54:19,984 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:19,987 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:20,004 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987, exclude list is [], retry=0 2024-11-13T19:54:20,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:20,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:20,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:20,044 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987 2024-11-13T19:54:20,045 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:20,095 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:54:20,133 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 394d753144b126e9496d21c901cb380c, NAME => 'testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:20,136 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,136 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:20,136 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,136 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,157 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,158 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName a 2024-11-13T19:54:20,158 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:20,159 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:20,160 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,161 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName b 2024-11-13T19:54:20,161 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:20,166 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:20,166 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,168 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName c 2024-11-13T19:54:20,168 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:20,168 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:20,169 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,170 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,171 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,172 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,173 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,173 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:20,178 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,180 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 394d753144b126e9496d21c901cb380c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66009039, jitterRate=-0.0163886696100235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:20,180 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:20,181 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 394d753144b126e9496d21c901cb380c: Running coprocessor pre-open hook at 1731527660136Writing region info on filesystem at 1731527660136Initializing all the Stores at 1731527660138 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527660138Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527660156 (+18 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527660156Cleaning up temporary data from old regions at 1731527660173 (+17 ms)Running coprocessor post-open hooks at 1731527660180 (+7 ms)Region opened successfully at 1731527660181 (+1 ms) 2024-11-13T19:54:20,223 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 394d753144b126e9496d21c901cb380c 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T19:54:20,224 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:21,225 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:22,226 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:23,227 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:23,705 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:54:24,228 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:25,131 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T19:54:25,131 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:25,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T19:54:25,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:25,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T19:54:25,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:25,229 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:25,638 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:54:26,229 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:27,230 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:28,231 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:29,231 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 394d753144b126e9496d21c901cb380c/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:29,233 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 394d753144b126e9496d21c901cb380c: 2024-11-13T19:54:29,233 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:29,253 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 394d753144b126e9496d21c901cb380c: 2024-11-13T19:54:29,253 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T19:54:29,253 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 394d753144b126e9496d21c901cb380c, disabling compactions & flushes 2024-11-13T19:54:29,253 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:29,253 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:29,253 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. after waiting 0 ms 2024-11-13T19:54:29,253 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:29,254 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:29,254 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. 2024-11-13T19:54:29,254 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 394d753144b126e9496d21c901cb380c: Waiting for close lock at 1731527669253Running coprocessor pre-close hooks at 1731527669253Disabling compacts and flushes for region at 1731527669253Disabling writes for close at 1731527669253Writing region close event to WAL at 1731527669254 (+1 ms)Running coprocessor post-close hooks at 1731527669254Closed at 1731527669254 2024-11-13T19:54:29,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741903_1081 (size=4236) 2024-11-13T19:54:29,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741903_1081 (size=4236) 2024-11-13T19:54:29,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741903_1081 (size=4236) 2024-11-13T19:54:29,289 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987, size=4.1 K (4236bytes) 2024-11-13T19:54:29,289 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987 2024-11-13T19:54:29,289 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987 after 0ms 2024-11-13T19:54:29,327 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:29,328 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987 took 38ms 2024-11-13T19:54:29,332 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987 so closing down 2024-11-13T19:54:29,332 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:29,338 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731527659987.temp 2024-11-13T19:54:29,377 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000004-wal.1731527659987.temp 2024-11-13T19:54:29,379 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:29,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741904_1082 (size=3245) 2024-11-13T19:54:29,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741904_1082 (size=3245) 2024-11-13T19:54:29,405 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000004-wal.1731527659987.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:29,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741904_1082 (size=3245) 2024-11-13T19:54:29,407 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000004-wal.1731527659987.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000026 2024-11-13T19:54:29,407 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 78 ms; skipped=3; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-13T19:54:29,407 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987, journal: Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987, size=4.1 K (4236bytes) at 1731527669289Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987 so closing down at 1731527669332 (+43 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000004-wal.1731527659987.temp at 1731527669377 (+45 ms)3 split writer threads finished at 1731527669379 (+2 ms)Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000004-wal.1731527659987.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731527669405 (+26 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000004-wal.1731527659987.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000026 at 1731527669407 (+2 ms)Processed 23 edits across 1 Regions in 78 ms; skipped=3; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1731527669407 2024-11-13T19:54:29,409 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527659987 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527659987 2024-11-13T19:54:29,410 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000026 2024-11-13T19:54:29,411 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:29,414 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:29,437 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527669416, exclude list is [], retry=0 2024-11-13T19:54:29,442 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:29,442 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:29,443 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:29,486 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527669416 2024-11-13T19:54:29,488 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:29,489 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 394d753144b126e9496d21c901cb380c, NAME => 'testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:29,490 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,490 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:29,490 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,490 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,493 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,494 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName a 2024-11-13T19:54:29,494 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:29,495 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:29,495 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,496 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName b 2024-11-13T19:54:29,496 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:29,497 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:29,497 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,498 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 394d753144b126e9496d21c901cb380c columnFamilyName c 2024-11-13T19:54:29,498 DEBUG [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:29,500 INFO [StoreOpener-394d753144b126e9496d21c901cb380c-1 {}] regionserver.HStore(327): Store=394d753144b126e9496d21c901cb380c/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:29,500 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,502 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,503 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000026 2024-11-13T19:54:29,535 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:29,537 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000026 2024-11-13T19:54:29,537 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 394d753144b126e9496d21c901cb380c 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T19:54:29,557 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/a/4a2713afe9de4f32928fc0d96be75ed4 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731527669240/Put/seqid=0 2024-11-13T19:54:29,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741906_1084 (size=5523) 2024-11-13T19:54:29,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741906_1084 (size=5523) 2024-11-13T19:54:29,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741906_1084 (size=5523) 2024-11-13T19:54:29,568 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/a/4a2713afe9de4f32928fc0d96be75ed4 2024-11-13T19:54:29,590 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/b/875456d0bc5e4e4c8d33f839f37f049c is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731527669234/Put/seqid=0 2024-11-13T19:54:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741907_1085 (size=5524) 2024-11-13T19:54:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741907_1085 (size=5524) 2024-11-13T19:54:29,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741907_1085 (size=5524) 2024-11-13T19:54:29,598 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/b/875456d0bc5e4e4c8d33f839f37f049c 2024-11-13T19:54:29,618 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/c/f99991b4a27742efac7b4d570e9d6cc4 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731527669237/Put/seqid=0 2024-11-13T19:54:29,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741908_1086 (size=5457) 2024-11-13T19:54:29,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741908_1086 (size=5457) 2024-11-13T19:54:29,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741908_1086 (size=5457) 2024-11-13T19:54:29,632 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/c/f99991b4a27742efac7b4d570e9d6cc4 2024-11-13T19:54:29,639 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/a/4a2713afe9de4f32928fc0d96be75ed4 as hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/a/4a2713afe9de4f32928fc0d96be75ed4 2024-11-13T19:54:29,645 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/a/4a2713afe9de4f32928fc0d96be75ed4, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T19:54:29,646 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/b/875456d0bc5e4e4c8d33f839f37f049c as hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/b/875456d0bc5e4e4c8d33f839f37f049c 2024-11-13T19:54:29,651 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/b/875456d0bc5e4e4c8d33f839f37f049c, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T19:54:29,652 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/.tmp/c/f99991b4a27742efac7b4d570e9d6cc4 as hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/c/f99991b4a27742efac7b4d570e9d6cc4 2024-11-13T19:54:29,657 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/c/f99991b4a27742efac7b4d570e9d6cc4, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T19:54:29,658 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 394d753144b126e9496d21c901cb380c in 120ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T19:54:29,658 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/0000000000000000026 2024-11-13T19:54:29,660 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,660 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,660 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:29,662 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,664 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsAfterAbortingFlush/394d753144b126e9496d21c901cb380c/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T19:54:29,665 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 394d753144b126e9496d21c901cb380c; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59333317, jitterRate=-0.11586467921733856}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:29,666 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 394d753144b126e9496d21c901cb380c 2024-11-13T19:54:29,666 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 394d753144b126e9496d21c901cb380c: Running coprocessor pre-open hook at 1731527669491Writing region info on filesystem at 1731527669491Initializing all the Stores at 1731527669492 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527669492Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527669492Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527669492Obtaining lock to block concurrent updates at 1731527669537 (+45 ms)Preparing flush snapshotting stores in 394d753144b126e9496d21c901cb380c at 1731527669537Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731527669538 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1731527659301.394d753144b126e9496d21c901cb380c. at 1731527669538Flushing 394d753144b126e9496d21c901cb380c/a: creating writer at 1731527669538Flushing 394d753144b126e9496d21c901cb380c/a: appending metadata at 1731527669556 (+18 ms)Flushing 394d753144b126e9496d21c901cb380c/a: closing flushed file at 1731527669556Flushing 394d753144b126e9496d21c901cb380c/b: creating writer at 1731527669574 (+18 ms)Flushing 394d753144b126e9496d21c901cb380c/b: appending metadata at 1731527669590 (+16 ms)Flushing 394d753144b126e9496d21c901cb380c/b: closing flushed file at 1731527669590Flushing 394d753144b126e9496d21c901cb380c/c: creating writer at 1731527669603 (+13 ms)Flushing 394d753144b126e9496d21c901cb380c/c: appending metadata at 1731527669618 (+15 ms)Flushing 394d753144b126e9496d21c901cb380c/c: closing flushed file at 1731527669618Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@338939b8: reopening flushed file at 1731527669638 (+20 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d337a8e: reopening flushed file at 1731527669645 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@426fc06b: reopening flushed file at 1731527669651 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 394d753144b126e9496d21c901cb380c in 120ms, sequenceid=26, compaction requested=false; wal=null at 1731527669658 (+7 ms)Cleaning up temporary data from old regions at 1731527669660 (+2 ms)Running coprocessor post-open hooks at 1731527669666 (+6 ms)Region opened successfully at 1731527669666 2024-11-13T19:54:29,688 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414 (was 407) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:52276 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:60632 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:42542 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:52290 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:42516 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:60624 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1180 (was 1112) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=470 (was 503), ProcessCount=11 (was 11), AvailableMemoryMB=2563 (was 2281) - AvailableMemoryMB LEAK? - 2024-11-13T19:54:29,688 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-13T19:54:29,703 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=414, OpenFileDescriptor=1180, MaxFileDescriptor=1048576, SystemLoadAverage=470, ProcessCount=11, AvailableMemoryMB=2562 2024-11-13T19:54:29,703 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1180 is superior to 1024 2024-11-13T19:54:29,725 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:29,727 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:29,728 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:29,731 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-08625281, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-08625281, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:29,745 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-08625281/hregion-08625281.1731527669731, exclude list is [], retry=0 2024-11-13T19:54:29,747 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:29,748 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:29,748 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:29,788 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-08625281/hregion-08625281.1731527669731 2024-11-13T19:54:29,789 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:29,789 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 518f061b394ad738d2bad1ae0a66d4e6, NAME => 'testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:29,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741910_1088 (size=61) 2024-11-13T19:54:29,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741910_1088 (size=61) 2024-11-13T19:54:29,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741910_1088 (size=61) 2024-11-13T19:54:29,799 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:29,801 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,802 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518f061b394ad738d2bad1ae0a66d4e6 columnFamilyName a 2024-11-13T19:54:29,802 DEBUG [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:29,803 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(327): Store=518f061b394ad738d2bad1ae0a66d4e6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:29,803 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,804 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,804 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,804 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,806 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,808 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:29,808 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 518f061b394ad738d2bad1ae0a66d4e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67975340, jitterRate=0.012911498546600342}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:54:29,809 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 518f061b394ad738d2bad1ae0a66d4e6: Writing region info on filesystem at 1731527669799Initializing all the Stores at 1731527669800 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527669800Cleaning up temporary data from old regions at 1731527669804 (+4 ms)Region opened successfully at 1731527669809 (+5 ms) 2024-11-13T19:54:29,809 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 518f061b394ad738d2bad1ae0a66d4e6, disabling compactions & flushes 2024-11-13T19:54:29,809 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,809 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,809 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. after waiting 0 ms 2024-11-13T19:54:29,809 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,809 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,810 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 518f061b394ad738d2bad1ae0a66d4e6: Waiting for close lock at 1731527669809Disabling compacts and flushes for region at 1731527669809Disabling writes for close at 1731527669809Writing region close event to WAL at 1731527669809Closed at 1731527669809 2024-11-13T19:54:29,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741909_1087 (size=175) 2024-11-13T19:54:29,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741909_1087 (size=175) 2024-11-13T19:54:29,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741909_1087 (size=175) 2024-11-13T19:54:29,815 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:29,815 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-08625281:(num 1731527669731) 2024-11-13T19:54:29,815 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:29,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:29,832 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817, exclude list is [], retry=0 2024-11-13T19:54:29,835 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:29,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:29,836 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:29,873 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817 2024-11-13T19:54:29,873 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:29,873 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 518f061b394ad738d2bad1ae0a66d4e6, NAME => 'testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:29,874 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:29,874 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,874 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,875 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,876 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518f061b394ad738d2bad1ae0a66d4e6 columnFamilyName a 2024-11-13T19:54:29,876 DEBUG [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:29,877 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(327): Store=518f061b394ad738d2bad1ae0a66d4e6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:29,877 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,877 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,878 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,879 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,879 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,881 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:29,881 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 518f061b394ad738d2bad1ae0a66d4e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64369007, jitterRate=-0.04082705080509186}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:54:29,882 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 518f061b394ad738d2bad1ae0a66d4e6: Writing region info on filesystem at 1731527669874Initializing all the Stores at 1731527669875 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527669875Cleaning up temporary data from old regions at 1731527669879 (+4 ms)Region opened successfully at 1731527669882 (+3 ms) 2024-11-13T19:54:29,934 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 518f061b394ad738d2bad1ae0a66d4e6, disabling compactions & flushes 2024-11-13T19:54:29,934 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,934 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,934 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. after waiting 0 ms 2024-11-13T19:54:29,934 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,936 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,936 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. 2024-11-13T19:54:29,936 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 518f061b394ad738d2bad1ae0a66d4e6: Waiting for close lock at 1731527669934Disabling compacts and flushes for region at 1731527669934Disabling writes for close at 1731527669934Writing region close event to WAL at 1731527669936 (+2 ms)Closed at 1731527669936 2024-11-13T19:54:29,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741911_1089 (size=1795) 2024-11-13T19:54:29,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741911_1089 (size=1795) 2024-11-13T19:54:29,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741911_1089 (size=1795) 2024-11-13T19:54:29,965 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817, size=1.8 K (1795bytes) 2024-11-13T19:54:29,965 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817 2024-11-13T19:54:29,966 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817 after 0ms 2024-11-13T19:54:30,000 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:30,000 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817 took 35ms 2024-11-13T19:54:30,004 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817 so closing down 2024-11-13T19:54:30,004 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:30,006 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731527669817.temp 2024-11-13T19:54:30,040 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000003-wal.1731527669817.temp 2024-11-13T19:54:30,040 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741912_1090 (size=1795) 2024-11-13T19:54:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741912_1090 (size=1795) 2024-11-13T19:54:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741912_1090 (size=1795) 2024-11-13T19:54:30,057 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000003-wal.1731527669817.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:30,058 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000003-wal.1731527669817.temp to hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000012 2024-11-13T19:54:30,059 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-13T19:54:30,059 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817, journal: Splitting hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817, size=1.8 K (1795bytes) at 1731527669965Finishing writing output for hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817 so closing down at 1731527670004 (+39 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000003-wal.1731527669817.temp at 1731527670040 (+36 ms)3 split writer threads finished at 1731527670040Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000003-wal.1731527669817.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731527670057 (+17 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000003-wal.1731527669817.temp to hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000012 at 1731527670059 (+2 ms)Processed 10 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1731527670059 2024-11-13T19:54:30,061 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527669817 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527669817 2024-11-13T19:54:30,062 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000012 2024-11-13T19:54:30,104 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:30,577 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:30,581 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:30,601 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527670581, exclude list is [], retry=0 2024-11-13T19:54:30,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:30,605 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:30,606 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:30,667 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527670581 2024-11-13T19:54:30,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:30,668 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 518f061b394ad738d2bad1ae0a66d4e6, NAME => 'testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:30,668 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:30,668 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,668 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,677 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,680 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518f061b394ad738d2bad1ae0a66d4e6 columnFamilyName a 2024-11-13T19:54:30,680 DEBUG [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:30,681 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(327): Store=518f061b394ad738d2bad1ae0a66d4e6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:30,682 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,683 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,685 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,686 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000012 2024-11-13T19:54:30,728 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:30,730 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000012 2024-11-13T19:54:30,730 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 518f061b394ad738d2bad1ae0a66d4e6 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T19:54:30,759 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/.tmp/a/2772e2e4a7ea4f8b88d8c911af632ed6 is 79, key is testDatalossWhenInputError/a:x0/1731527669882/Put/seqid=0 2024-11-13T19:54:30,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741914_1092 (size=5808) 2024-11-13T19:54:30,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741914_1092 (size=5808) 2024-11-13T19:54:30,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741914_1092 (size=5808) 2024-11-13T19:54:30,771 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/.tmp/a/2772e2e4a7ea4f8b88d8c911af632ed6 2024-11-13T19:54:30,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/.tmp/a/2772e2e4a7ea4f8b88d8c911af632ed6 as hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/a/2772e2e4a7ea4f8b88d8c911af632ed6 2024-11-13T19:54:30,797 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/a/2772e2e4a7ea4f8b88d8c911af632ed6, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T19:54:30,797 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 518f061b394ad738d2bad1ae0a66d4e6 in 67ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T19:54:30,798 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/0000000000000000012 2024-11-13T19:54:30,799 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,799 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,802 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,806 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T19:54:30,807 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 518f061b394ad738d2bad1ae0a66d4e6; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75291363, jitterRate=0.12192873656749725}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:54:30,808 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 518f061b394ad738d2bad1ae0a66d4e6: Writing region info on filesystem at 1731527670668Initializing all the Stores at 1731527670675 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527670675Obtaining lock to block concurrent updates at 1731527670730 (+55 ms)Preparing flush snapshotting stores in 518f061b394ad738d2bad1ae0a66d4e6 at 1731527670730Finished memstore snapshotting testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731527670730Flushing stores of testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6. at 1731527670730Flushing 518f061b394ad738d2bad1ae0a66d4e6/a: creating writer at 1731527670730Flushing 518f061b394ad738d2bad1ae0a66d4e6/a: appending metadata at 1731527670758 (+28 ms)Flushing 518f061b394ad738d2bad1ae0a66d4e6/a: closing flushed file at 1731527670758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a07d277: reopening flushed file at 1731527670782 (+24 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 518f061b394ad738d2bad1ae0a66d4e6 in 67ms, sequenceid=12, compaction requested=false; wal=null at 1731527670797 (+15 ms)Cleaning up temporary data from old regions at 1731527670799 (+2 ms)Region opened successfully at 1731527670808 (+9 ms) 2024-11-13T19:54:30,824 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 518f061b394ad738d2bad1ae0a66d4e6, NAME => 'testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:30,824 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731527669726.518f061b394ad738d2bad1ae0a66d4e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:30,824 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,824 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,826 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,827 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 518f061b394ad738d2bad1ae0a66d4e6 columnFamilyName a 2024-11-13T19:54:30,827 DEBUG [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:30,837 DEBUG [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/a/2772e2e4a7ea4f8b88d8c911af632ed6 2024-11-13T19:54:30,838 INFO [StoreOpener-518f061b394ad738d2bad1ae0a66d4e6-1 {}] regionserver.HStore(327): Store=518f061b394ad738d2bad1ae0a66d4e6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:30,838 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,839 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,842 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,842 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,842 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,846 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 518f061b394ad738d2bad1ae0a66d4e6 2024-11-13T19:54:30,849 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testDatalossWhenInputError/518f061b394ad738d2bad1ae0a66d4e6/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T19:54:30,850 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 518f061b394ad738d2bad1ae0a66d4e6; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74238914, jitterRate=0.10624602437019348}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T19:54:30,850 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 518f061b394ad738d2bad1ae0a66d4e6: Writing region info on filesystem at 1731527670824Initializing all the Stores at 1731527670825 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527670826 (+1 ms)Cleaning up temporary data from old regions at 1731527670842 (+16 ms)Region opened successfully at 1731527670850 (+8 ms) 2024-11-13T19:54:30,871 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=424 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:52276 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:60632 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:42542 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:52368 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:42570 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:60688 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1458969915-172.17.0.2-1731527631214:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1180) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=470 (was 470), ProcessCount=11 (was 11), AvailableMemoryMB=2771 (was 2562) - AvailableMemoryMB LEAK? - 2024-11-13T19:54:30,872 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T19:54:30,887 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=470, ProcessCount=11, AvailableMemoryMB=2768 2024-11-13T19:54:30,887 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T19:54:30,903 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:30,906 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:30,906 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:30,910 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35564167, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-35564167, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:30,930 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35564167/hregion-35564167.1731527670910, exclude list is [], retry=0 2024-11-13T19:54:30,934 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:30,938 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:30,939 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:30,995 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35564167/hregion-35564167.1731527670910 2024-11-13T19:54:30,996 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:30,996 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1560ec028eadcf30590e35ebd2fd73c0, NAME => 'testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:31,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741916_1094 (size=63) 2024-11-13T19:54:31,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741916_1094 (size=63) 2024-11-13T19:54:31,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741916_1094 (size=63) 2024-11-13T19:54:31,036 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:31,041 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,043 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName a 2024-11-13T19:54:31,044 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:31,045 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:31,045 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,051 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName b 2024-11-13T19:54:31,051 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:31,051 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:31,052 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,053 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName c 2024-11-13T19:54:31,053 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:31,054 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:31,054 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,055 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,056 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,057 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,057 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,058 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:31,060 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,066 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:31,066 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1560ec028eadcf30590e35ebd2fd73c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73597366, jitterRate=0.0966862142086029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1560ec028eadcf30590e35ebd2fd73c0: Writing region info on filesystem at 1731527671036Initializing all the Stores at 1731527671037 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527671037Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527671041 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527671041Cleaning up temporary data from old regions at 1731527671057 (+16 ms)Region opened successfully at 1731527671067 (+10 ms) 2024-11-13T19:54:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1560ec028eadcf30590e35ebd2fd73c0, disabling compactions & flushes 2024-11-13T19:54:31,067 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. after waiting 0 ms 2024-11-13T19:54:31,067 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:31,068 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:31,068 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1560ec028eadcf30590e35ebd2fd73c0: Waiting for close lock at 1731527671067Disabling compacts and flushes for region at 1731527671067Disabling writes for close at 1731527671067Writing region close event to WAL at 1731527671068 (+1 ms)Closed at 1731527671068 2024-11-13T19:54:31,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741915_1093 (size=175) 2024-11-13T19:54:31,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741915_1093 (size=175) 2024-11-13T19:54:31,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741915_1093 (size=175) 2024-11-13T19:54:31,083 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:31,083 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35564167:(num 1731527670910) 2024-11-13T19:54:31,083 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:31,085 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:31,099 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085, exclude list is [], retry=0 2024-11-13T19:54:31,102 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:31,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:31,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:31,151 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 2024-11-13T19:54:31,152 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605)] 2024-11-13T19:54:31,152 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1560ec028eadcf30590e35ebd2fd73c0, NAME => 'testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:31,152 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:31,152 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,152 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,154 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,155 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName a 2024-11-13T19:54:31,155 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:31,156 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:31,156 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,165 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName b 2024-11-13T19:54:31,166 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:31,168 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:31,168 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,169 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName c 2024-11-13T19:54:31,169 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:31,169 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:31,170 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,170 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,172 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,173 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,173 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,174 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:31,175 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:31,177 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1560ec028eadcf30590e35ebd2fd73c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63156739, jitterRate=-0.05889125168323517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:31,178 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1560ec028eadcf30590e35ebd2fd73c0: Writing region info on filesystem at 1731527671152Initializing all the Stores at 1731527671153 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527671153Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527671154 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527671154Cleaning up temporary data from old regions at 1731527671173 (+19 ms)Region opened successfully at 1731527671178 (+5 ms) 2024-11-13T19:54:31,185 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731527671184/Put/seqid=0 2024-11-13T19:54:31,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741918_1096 (size=4875) 2024-11-13T19:54:31,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741918_1096 (size=4875) 2024-11-13T19:54:31,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741918_1096 (size=4875) 2024-11-13T19:54:31,223 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731527671222/Put/seqid=0 2024-11-13T19:54:31,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741919_1097 (size=4875) 2024-11-13T19:54:31,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741919_1097 (size=4875) 2024-11-13T19:54:31,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741919_1097 (size=4875) 2024-11-13T19:54:31,233 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731527671233/Put/seqid=0 2024-11-13T19:54:31,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741920_1098 (size=4875) 2024-11-13T19:54:31,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741920_1098 (size=4875) 2024-11-13T19:54:31,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741920_1098 (size=4875) 2024-11-13T19:54:31,244 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 1560ec028eadcf30590e35ebd2fd73c0/a 2024-11-13T19:54:31,254 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T19:54:31,254 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T19:54:31,254 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 1560ec028eadcf30590e35ebd2fd73c0/a 2024-11-13T19:54:31,262 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T19:54:31,262 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T19:54:31,262 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 1560ec028eadcf30590e35ebd2fd73c0/a 2024-11-13T19:54:31,266 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T19:54:31,266 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T19:54:31,266 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1560ec028eadcf30590e35ebd2fd73c0 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T19:54:31,288 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/.tmp/a/321f686dad074ed1a2e5afff6a99c7e3 is 55, key is testCompactedBulkLoadedFiles/a:a/1731527671178/Put/seqid=0 2024-11-13T19:54:31,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741921_1099 (size=5107) 2024-11-13T19:54:31,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741921_1099 (size=5107) 2024-11-13T19:54:31,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741921_1099 (size=5107) 2024-11-13T19:54:31,297 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/.tmp/a/321f686dad074ed1a2e5afff6a99c7e3 2024-11-13T19:54:31,303 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/.tmp/a/321f686dad074ed1a2e5afff6a99c7e3 as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3 2024-11-13T19:54:31,309 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T19:54:31,313 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 1560ec028eadcf30590e35ebd2fd73c0 in 47ms, sequenceid=4, compaction requested=false 2024-11-13T19:54:31,313 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1560ec028eadcf30590e35ebd2fd73c0: 2024-11-13T19:54:31,325 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_ 2024-11-13T19:54:31,329 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_ 2024-11-13T19:54:31,331 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_ 2024-11-13T19:54:31,334 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile0 into 1560ec028eadcf30590e35ebd2fd73c0/a as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_ - updating store file list. 2024-11-13T19:54:31,347 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:31,348 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_ into 1560ec028eadcf30590e35ebd2fd73c0/a 2024-11-13T19:54:31,348 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile0 into 1560ec028eadcf30590e35ebd2fd73c0/a (new location: hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_) 2024-11-13T19:54:31,349 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile1 into 1560ec028eadcf30590e35ebd2fd73c0/a as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_ - updating store file list. 2024-11-13T19:54:31,357 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:31,357 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_ into 1560ec028eadcf30590e35ebd2fd73c0/a 2024-11-13T19:54:31,357 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile1 into 1560ec028eadcf30590e35ebd2fd73c0/a (new location: hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_) 2024-11-13T19:54:31,359 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile2 into 1560ec028eadcf30590e35ebd2fd73c0/a as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_ - updating store file list. 2024-11-13T19:54:31,364 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:31,364 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_ into 1560ec028eadcf30590e35ebd2fd73c0/a 2024-11-13T19:54:31,364 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:39891/hbase/testCompactedBulkLoadedFiles/hfile2 into 1560ec028eadcf30590e35ebd2fd73c0/a (new location: hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_) 2024-11-13T19:54:31,373 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:54:31,374 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1560ec028eadcf30590e35ebd2fd73c0/a is initiating major compaction (all files) 2024-11-13T19:54:31,374 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1560ec028eadcf30590e35ebd2fd73c0/a in testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:31,374 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_] into tmpdir=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/.tmp, totalSize=19.3 K 2024-11-13T19:54:31,375 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 321f686dad074ed1a2e5afff6a99c7e3, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731527671178 2024-11-13T19:54:31,375 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T19:54:31,376 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T19:54:31,376 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T19:54:31,396 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/.tmp/a/ff31a7dc6eb348dd89251d6d2a0370c3 is 55, key is testCompactedBulkLoadedFiles/a:a/1731527671178/Put/seqid=0 2024-11-13T19:54:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741922_1100 (size=6154) 2024-11-13T19:54:31,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741922_1100 (size=6154) 2024-11-13T19:54:31,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741922_1100 (size=6154) 2024-11-13T19:54:31,465 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/.tmp/a/ff31a7dc6eb348dd89251d6d2a0370c3 as hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ff31a7dc6eb348dd89251d6d2a0370c3 2024-11-13T19:54:31,473 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 1560ec028eadcf30590e35ebd2fd73c0/a of 1560ec028eadcf30590e35ebd2fd73c0 into ff31a7dc6eb348dd89251d6d2a0370c3(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:54:31,473 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1560ec028eadcf30590e35ebd2fd73c0: 2024-11-13T19:54:31,473 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T19:54:31,473 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T19:54:31,519 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085, size=0 (0bytes) 2024-11-13T19:54:31,519 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 might be still open, length is 0 2024-11-13T19:54:31,519 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 2024-11-13T19:54:31,520 WARN [IPC Server handler 1 on default port 39891 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-13T19:54:31,521 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 after 2ms 2024-11-13T19:54:32,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:42606 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33541:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42606 dst: /127.0.0.1:33541 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33541 remote=/127.0.0.1:42606]. Total timeout mills is 60000, 58984 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:32,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:52404 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33527:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52404 dst: /127.0.0.1:33527 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:32,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:60736 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60736 dst: /127.0.0.1:43261 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:32,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741917_1101 (size=1794) 2024-11-13T19:54:32,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741917_1101 (size=1794) 2024-11-13T19:54:32,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741917_1101 (size=1794) 2024-11-13T19:54:34,756 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:54:35,131 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T19:54:35,131 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:35,132 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T19:54:35,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T19:54:35,522 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 after 4003ms 2024-11-13T19:54:35,565 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:35,565 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 took 4047ms 2024-11-13T19:54:35,568 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085; continuing. 2024-11-13T19:54:35,568 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 so closing down 2024-11-13T19:54:35,568 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:35,570 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731527671085.temp 2024-11-13T19:54:35,596 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000003-wal.1731527671085.temp 2024-11-13T19:54:35,596 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:35,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741923_1102 (size=800) 2024-11-13T19:54:35,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741923_1102 (size=800) 2024-11-13T19:54:35,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741923_1102 (size=800) 2024-11-13T19:54:35,604 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000003-wal.1731527671085.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:35,606 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000003-wal.1731527671085.temp to hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000008 2024-11-13T19:54:35,606 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T19:54:35,606 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085, journal: Splitting hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085, size=0 (0bytes) at 1731527671519Finishing writing output for hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 so closing down at 1731527675568 (+4049 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000003-wal.1731527671085.temp at 1731527675596 (+28 ms)3 split writer threads finished at 1731527675596Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000003-wal.1731527671085.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731527675604 (+8 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000003-wal.1731527671085.temp to hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000008 at 1731527675606 (+2 ms)Processed 5 edits across 1 Regions in 40 ms; skipped=3; WAL=hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085, size=0, length=0, corrupted=false, cancelled=false at 1731527675606 2024-11-13T19:54:35,608 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527671085 2024-11-13T19:54:35,609 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000008 2024-11-13T19:54:35,609 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:35,610 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:35,624 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527675611, exclude list is [], retry=0 2024-11-13T19:54:35,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:35,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:35,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:35,657 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527675611 2024-11-13T19:54:35,658 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:35,658 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1560ec028eadcf30590e35ebd2fd73c0, NAME => 'testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:35,658 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:35,658 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,658 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,660 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,661 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName a 2024-11-13T19:54:35,661 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,668 DEBUG [StoreFileOpener-1560ec028eadcf30590e35ebd2fd73c0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:35,669 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_ 2024-11-13T19:54:35,673 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3 2024-11-13T19:54:35,677 DEBUG [StoreFileOpener-1560ec028eadcf30590e35ebd2fd73c0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:35,677 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_ 2024-11-13T19:54:35,681 DEBUG [StoreFileOpener-1560ec028eadcf30590e35ebd2fd73c0-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T19:54:35,685 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_ 2024-11-13T19:54:35,689 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ff31a7dc6eb348dd89251d6d2a0370c3 2024-11-13T19:54:35,689 WARN [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@66ee5ad9 2024-11-13T19:54:35,689 WARN [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@66ee5ad9 2024-11-13T19:54:35,689 WARN [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@66ee5ad9 2024-11-13T19:54:35,690 WARN [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@66ee5ad9 2024-11-13T19:54:35,690 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_] to archive 2024-11-13T19:54:35,691 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:54:35,693 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_ to hdfs://localhost:39891/hbase/archive/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_ 2024-11-13T19:54:35,695 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3 to hdfs://localhost:39891/hbase/archive/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/321f686dad074ed1a2e5afff6a99c7e3 2024-11-13T19:54:35,696 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_ to hdfs://localhost:39891/hbase/archive/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_ 2024-11-13T19:54:35,697 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_ to hdfs://localhost:39891/hbase/archive/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/a/ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_ 2024-11-13T19:54:35,697 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,698 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,698 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName b 2024-11-13T19:54:35,699 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,699 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,699 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,700 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1560ec028eadcf30590e35ebd2fd73c0 columnFamilyName c 2024-11-13T19:54:35,700 DEBUG [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,701 INFO [StoreOpener-1560ec028eadcf30590e35ebd2fd73c0-1 {}] regionserver.HStore(327): Store=1560ec028eadcf30590e35ebd2fd73c0/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,701 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,702 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,704 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000008 2024-11-13T19:54:35,732 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:35,734 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 1560ec028eadcf30590e35ebd2fd73c0 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "1560ec028eadcf30590e35ebd2fd73c0" family_name: "a" compaction_input: "321f686dad074ed1a2e5afff6a99c7e3" compaction_input: "a9019b4eaf1947a1a24e3a6685ebe0fd_SeqId_4_" compaction_input: "ef5ce56b74864569bcbeb9f79adae84a_SeqId_4_" compaction_input: "1fb13f6bd16c4bd0aae2de1419e70c8a_SeqId_4_" compaction_output: "ff31a7dc6eb348dd89251d6d2a0370c3" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T19:54:35,734 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T19:54:35,734 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000008 2024-11-13T19:54:35,735 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/0000000000000000008 2024-11-13T19:54:35,737 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,737 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,738 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:35,739 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1560ec028eadcf30590e35ebd2fd73c0 2024-11-13T19:54:35,742 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testCompactedBulkLoadedFiles/1560ec028eadcf30590e35ebd2fd73c0/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T19:54:35,743 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1560ec028eadcf30590e35ebd2fd73c0; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71699164, jitterRate=0.0684008002281189}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:35,743 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1560ec028eadcf30590e35ebd2fd73c0: Writing region info on filesystem at 1731527675658Initializing all the Stores at 1731527675659 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675659Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675660 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675660Cleaning up temporary data from old regions at 1731527675737 (+77 ms)Region opened successfully at 1731527675743 (+6 ms) 2024-11-13T19:54:35,746 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1560ec028eadcf30590e35ebd2fd73c0, disabling compactions & flushes 2024-11-13T19:54:35,746 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:35,746 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:35,746 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. after waiting 0 ms 2024-11-13T19:54:35,746 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:35,747 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731527670904.1560ec028eadcf30590e35ebd2fd73c0. 2024-11-13T19:54:35,747 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1560ec028eadcf30590e35ebd2fd73c0: Waiting for close lock at 1731527675746Disabling compacts and flushes for region at 1731527675746Disabling writes for close at 1731527675746Writing region close event to WAL at 1731527675747 (+1 ms)Closed at 1731527675747 2024-11-13T19:54:35,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741924_1103 (size=175) 2024-11-13T19:54:35,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741924_1103 (size=175) 2024-11-13T19:54:35,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741924_1103 (size=175) 2024-11-13T19:54:35,754 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:35,754 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731527675611) 2024-11-13T19:54:35,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=434 (was 424) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-788643571_22 at /127.0.0.1:53770 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:39891 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1323050754) connection to localhost/127.0.0.1:39891 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-788643571_22 at /127.0.0.1:39840 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1340 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=481 (was 470) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2293 (was 2768) 2024-11-13T19:54:35,771 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1340 is superior to 1024 2024-11-13T19:54:35,784 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=434, OpenFileDescriptor=1340, MaxFileDescriptor=1048576, SystemLoadAverage=481, ProcessCount=11, AvailableMemoryMB=2292 2024-11-13T19:54:35,784 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1340 is superior to 1024 2024-11-13T19:54:35,798 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:35,800 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:54:35,801 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:54:35,803 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44854279, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/hregion-44854279, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:35,817 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44854279/hregion-44854279.1731527675804, exclude list is [], retry=0 2024-11-13T19:54:35,820 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:35,825 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:35,826 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:35,866 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44854279/hregion-44854279.1731527675804 2024-11-13T19:54:35,867 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:46605:46605)] 2024-11-13T19:54:35,867 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 895b71ec5b37de3f74e3ff3a2ca95081, NAME => 'testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39891/hbase 2024-11-13T19:54:35,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741926_1105 (size=67) 2024-11-13T19:54:35,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741926_1105 (size=67) 2024-11-13T19:54:35,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741926_1105 (size=67) 2024-11-13T19:54:35,877 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:35,878 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,880 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName a 2024-11-13T19:54:35,880 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,880 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,880 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,882 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName b 2024-11-13T19:54:35,882 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,882 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,882 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,883 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName c 2024-11-13T19:54:35,883 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,884 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,884 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,884 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,885 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,886 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,886 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,886 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:35,887 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,890 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:54:35,890 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895b71ec5b37de3f74e3ff3a2ca95081; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63864163, jitterRate=-0.048349812626838684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:35,891 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895b71ec5b37de3f74e3ff3a2ca95081: Writing region info on filesystem at 1731527675877Initializing all the Stores at 1731527675878 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675878Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675878Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675878Cleaning up temporary data from old regions at 1731527675886 (+8 ms)Region opened successfully at 1731527675890 (+4 ms) 2024-11-13T19:54:35,891 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 895b71ec5b37de3f74e3ff3a2ca95081, disabling compactions & flushes 2024-11-13T19:54:35,891 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:35,891 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:35,891 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. after waiting 0 ms 2024-11-13T19:54:35,891 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:35,891 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:35,891 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 895b71ec5b37de3f74e3ff3a2ca95081: Waiting for close lock at 1731527675891Disabling compacts and flushes for region at 1731527675891Disabling writes for close at 1731527675891Writing region close event to WAL at 1731527675891Closed at 1731527675891 2024-11-13T19:54:35,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741925_1104 (size=175) 2024-11-13T19:54:35,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741925_1104 (size=175) 2024-11-13T19:54:35,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741925_1104 (size=175) 2024-11-13T19:54:35,899 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:35,899 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44854279:(num 1731527675804) 2024-11-13T19:54:35,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:35,902 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:35,918 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902, exclude list is [], retry=0 2024-11-13T19:54:35,920 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:35,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:35,921 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:35,960 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902 2024-11-13T19:54:35,960 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:42917:42917)] 2024-11-13T19:54:35,960 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 895b71ec5b37de3f74e3ff3a2ca95081, NAME => 'testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:35,961 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:35,961 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,961 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,962 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,963 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName a 2024-11-13T19:54:35,963 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,964 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,964 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,965 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName b 2024-11-13T19:54:35,965 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,965 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,966 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,966 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName c 2024-11-13T19:54:35,966 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:35,967 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:35,969 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,970 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,971 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,972 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,972 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,973 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:35,975 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:35,976 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895b71ec5b37de3f74e3ff3a2ca95081; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59665181, jitterRate=-0.1109195202589035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:35,976 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895b71ec5b37de3f74e3ff3a2ca95081: Writing region info on filesystem at 1731527675961Initializing all the Stores at 1731527675962 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675962Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675962Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527675962Cleaning up temporary data from old regions at 1731527675972 (+10 ms)Region opened successfully at 1731527675976 (+4 ms) 2024-11-13T19:54:35,996 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 895b71ec5b37de3f74e3ff3a2ca95081 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T19:54:36,014 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/a/d54a9e27210d4a81a7caef8c62d631a5 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731527675976/Put/seqid=0 2024-11-13T19:54:36,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741928_1107 (size=5958) 2024-11-13T19:54:36,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741928_1107 (size=5958) 2024-11-13T19:54:36,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741928_1107 (size=5958) 2024-11-13T19:54:36,030 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/a/d54a9e27210d4a81a7caef8c62d631a5 2024-11-13T19:54:36,036 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/a/d54a9e27210d4a81a7caef8c62d631a5 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/a/d54a9e27210d4a81a7caef8c62d631a5 2024-11-13T19:54:36,041 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/a/d54a9e27210d4a81a7caef8c62d631a5, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T19:54:36,042 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 895b71ec5b37de3f74e3ff3a2ca95081 in 46ms, sequenceid=13, compaction requested=false 2024-11-13T19:54:36,042 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 895b71ec5b37de3f74e3ff3a2ca95081: 2024-11-13T19:54:36,064 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 895b71ec5b37de3f74e3ff3a2ca95081, disabling compactions & flushes 2024-11-13T19:54:36,064 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:36,064 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:36,064 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. after waiting 0 ms 2024-11-13T19:54:36,065 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:36,065 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:36,066 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:36,066 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 895b71ec5b37de3f74e3ff3a2ca95081: Waiting for close lock at 1731527676064Disabling compacts and flushes for region at 1731527676064Disabling writes for close at 1731527676064Writing region close event to WAL at 1731527676065 (+1 ms)Closed at 1731527676066 (+1 ms) 2024-11-13T19:54:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741927_1106 (size=6243) 2024-11-13T19:54:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741927_1106 (size=6243) 2024-11-13T19:54:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741927_1106 (size=6243) 2024-11-13T19:54:36,085 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902, size=6.1 K (6243bytes) 2024-11-13T19:54:36,085 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902 2024-11-13T19:54:36,085 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902 after 0ms 2024-11-13T19:54:36,112 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:36,112 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902 took 28ms 2024-11-13T19:54:36,114 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902 so closing down 2024-11-13T19:54:36,114 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:36,116 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731527675902.temp 2024-11-13T19:54:36,144 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000003-wal.1731527675902.temp 2024-11-13T19:54:36,144 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741929_1108 (size=5575) 2024-11-13T19:54:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741929_1108 (size=5575) 2024-11-13T19:54:36,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741929_1108 (size=5575) 2024-11-13T19:54:36,554 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000003-wal.1731527675902.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:36,555 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000003-wal.1731527675902.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000035 2024-11-13T19:54:36,555 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 442 ms; skipped=2; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-13T19:54:36,556 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902, journal: Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902, size=6.1 K (6243bytes) at 1731527676085Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902 so closing down at 1731527676114 (+29 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000003-wal.1731527675902.temp at 1731527676144 (+30 ms)3 split writer threads finished at 1731527676144Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000003-wal.1731527675902.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731527676554 (+410 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000003-wal.1731527675902.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000035 at 1731527676555 (+1 ms)Processed 32 edits across 1 Regions in 442 ms; skipped=2; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1731527676556 (+1 ms) 2024-11-13T19:54:36,557 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527675902 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527675902 2024-11-13T19:54:36,559 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000035 2024-11-13T19:54:36,559 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:36,561 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:36,574 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561, exclude list is [], retry=0 2024-11-13T19:54:36,576 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:36,577 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:36,577 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:36,604 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 2024-11-13T19:54:36,604 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:35753:35753),(127.0.0.1/127.0.0.1:46605:46605)] 2024-11-13T19:54:36,604 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 895b71ec5b37de3f74e3ff3a2ca95081, NAME => 'testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:54:36,604 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:36,604 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,604 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,606 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,606 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName a 2024-11-13T19:54:36,606 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:36,612 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/a/d54a9e27210d4a81a7caef8c62d631a5 2024-11-13T19:54:36,612 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:36,612 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,613 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName b 2024-11-13T19:54:36,613 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:36,614 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:36,614 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,614 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName c 2024-11-13T19:54:36,614 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:36,615 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:36,615 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,615 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,617 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,618 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000035 2024-11-13T19:54:36,645 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:36,647 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000035 2024-11-13T19:54:36,647 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 895b71ec5b37de3f74e3ff3a2ca95081 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T19:54:36,662 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/b/83786ac4cbb044c9a996bb2157f8611c is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731527676042/Put/seqid=0 2024-11-13T19:54:36,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741931_1110 (size=5958) 2024-11-13T19:54:36,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741931_1110 (size=5958) 2024-11-13T19:54:36,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741931_1110 (size=5958) 2024-11-13T19:54:36,671 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/b/83786ac4cbb044c9a996bb2157f8611c 2024-11-13T19:54:36,690 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/c/fb7dc3b525a1412583b2a87cfe1a47cb is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731527676053/Put/seqid=0 2024-11-13T19:54:36,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741932_1111 (size=5958) 2024-11-13T19:54:36,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741932_1111 (size=5958) 2024-11-13T19:54:36,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741932_1111 (size=5958) 2024-11-13T19:54:36,697 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/c/fb7dc3b525a1412583b2a87cfe1a47cb 2024-11-13T19:54:36,703 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/b/83786ac4cbb044c9a996bb2157f8611c as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/b/83786ac4cbb044c9a996bb2157f8611c 2024-11-13T19:54:36,707 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/b/83786ac4cbb044c9a996bb2157f8611c, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T19:54:36,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/c/fb7dc3b525a1412583b2a87cfe1a47cb as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/c/fb7dc3b525a1412583b2a87cfe1a47cb 2024-11-13T19:54:36,713 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/c/fb7dc3b525a1412583b2a87cfe1a47cb, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T19:54:36,713 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 895b71ec5b37de3f74e3ff3a2ca95081 in 66ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T19:54:36,714 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000035 2024-11-13T19:54:36,715 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,715 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,716 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:36,717 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:36,719 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T19:54:36,720 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895b71ec5b37de3f74e3ff3a2ca95081; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69459581, jitterRate=0.03502841293811798}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:36,720 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895b71ec5b37de3f74e3ff3a2ca95081: Writing region info on filesystem at 1731527676604Initializing all the Stores at 1731527676605 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527676605Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527676605Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527676605Obtaining lock to block concurrent updates at 1731527676647 (+42 ms)Preparing flush snapshotting stores in 895b71ec5b37de3f74e3ff3a2ca95081 at 1731527676647Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731527676647Flushing stores of testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. at 1731527676647Flushing 895b71ec5b37de3f74e3ff3a2ca95081/b: creating writer at 1731527676647Flushing 895b71ec5b37de3f74e3ff3a2ca95081/b: appending metadata at 1731527676661 (+14 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/b: closing flushed file at 1731527676661Flushing 895b71ec5b37de3f74e3ff3a2ca95081/c: creating writer at 1731527676676 (+15 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/c: appending metadata at 1731527676689 (+13 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/c: closing flushed file at 1731527676689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fd78c1d: reopening flushed file at 1731527676702 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65f5c1e2: reopening flushed file at 1731527676708 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 895b71ec5b37de3f74e3ff3a2ca95081 in 66ms, sequenceid=35, compaction requested=false; wal=null at 1731527676713 (+5 ms)Cleaning up temporary data from old regions at 1731527676715 (+2 ms)Region opened successfully at 1731527676720 (+5 ms) 2024-11-13T19:54:36,806 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561, size=0 (0bytes) 2024-11-13T19:54:36,806 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 might be still open, length is 0 2024-11-13T19:54:36,806 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 2024-11-13T19:54:36,807 WARN [IPC Server handler 3 on default port 39891 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-13T19:54:36,807 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 after 1ms 2024-11-13T19:54:38,489 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:39904 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43261:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39904 dst: /127.0.0.1:43261 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43261 remote=/127.0.0.1:39904]. Total timeout mills is 60000, 58263 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:38,490 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:53824 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33527:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53824 dst: /127.0.0.1:33527 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:38,490 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-228963406_22 at /127.0.0.1:51770 [Receiving block BP-1458969915-172.17.0.2-1731527631214:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33541:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51770 dst: /127.0.0.1:33541 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:38,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741930_1112 (size=5567) 2024-11-13T19:54:38,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741930_1112 (size=5567) 2024-11-13T19:54:38,611 INFO [master/54d0784d1943:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T19:54:38,611 INFO [master/54d0784d1943:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T19:54:40,808 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 after 4002ms 2024-11-13T19:54:40,853 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:40,853 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 took 4047ms 2024-11-13T19:54:40,856 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561; continuing. 2024-11-13T19:54:40,856 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 so closing down 2024-11-13T19:54:40,857 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T19:54:40,859 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731527676561.temp 2024-11-13T19:54:40,902 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000037-wal.1731527676561.temp 2024-11-13T19:54:40,903 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T19:54:40,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741933_1113 (size=5575) 2024-11-13T19:54:40,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741933_1113 (size=5575) 2024-11-13T19:54:40,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741933_1113 (size=5575) 2024-11-13T19:54:40,921 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000037-wal.1731527676561.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T19:54:40,926 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000037-wal.1731527676561.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000066 2024-11-13T19:54:40,926 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T19:54:40,926 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561, journal: Splitting hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561, size=0 (0bytes) at 1731527676806Finishing writing output for hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 so closing down at 1731527680856 (+4050 ms)Creating recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000037-wal.1731527676561.temp at 1731527680903 (+47 ms)3 split writer threads finished at 1731527680903Closed recovered edits writer path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000037-wal.1731527676561.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731527680921 (+18 ms)Rename recovered edits hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000037-wal.1731527676561.temp to hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000066 at 1731527680926 (+5 ms)Processed 30 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561, size=0, length=0, corrupted=false, cancelled=false at 1731527680926 2024-11-13T19:54:40,928 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 to hdfs://localhost:39891/hbase/oldWALs/wal.1731527676561 2024-11-13T19:54:40,929 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000066 2024-11-13T19:54:40,930 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T19:54:40,934 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:39891/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798, archiveDir=hdfs://localhost:39891/hbase/oldWALs, maxLogs=32 2024-11-13T19:54:40,952 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527680935, exclude list is [], retry=0 2024-11-13T19:54:40,954 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43261,DS-8dfaa49d-5bda-4241-b5ec-0a4746fe55e5,DISK] 2024-11-13T19:54:40,955 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33527,DS-7859c527-ec0a-41b8-80ba-8646ff04a519,DISK] 2024-11-13T19:54:40,955 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33541,DS-c00217f9-8b82-40cf-9fec-6506b10c0441,DISK] 2024-11-13T19:54:40,989 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527680935 2024-11-13T19:54:40,989 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42917:42917),(127.0.0.1/127.0.0.1:46605:46605),(127.0.0.1/127.0.0.1:35753:35753)] 2024-11-13T19:54:40,990 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:54:40,991 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:40,992 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName a 2024-11-13T19:54:40,992 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:41,000 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/a/d54a9e27210d4a81a7caef8c62d631a5 2024-11-13T19:54:41,000 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:41,000 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,002 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName b 2024-11-13T19:54:41,002 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:41,013 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/b/83786ac4cbb044c9a996bb2157f8611c 2024-11-13T19:54:41,013 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:41,013 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,014 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 895b71ec5b37de3f74e3ff3a2ca95081 columnFamilyName c 2024-11-13T19:54:41,014 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:54:41,020 DEBUG [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/c/fb7dc3b525a1412583b2a87cfe1a47cb 2024-11-13T19:54:41,020 INFO [StoreOpener-895b71ec5b37de3f74e3ff3a2ca95081-1 {}] regionserver.HStore(327): Store=895b71ec5b37de3f74e3ff3a2ca95081/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:54:41,020 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,023 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000066 2024-11-13T19:54:41,050 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-13T19:54:41,056 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000066 2024-11-13T19:54:41,056 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 895b71ec5b37de3f74e3ff3a2ca95081 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T19:54:41,078 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/a/d3cd9a25c7b84242b80368926f060822 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731527676726/Put/seqid=0 2024-11-13T19:54:41,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741935_1115 (size=5958) 2024-11-13T19:54:41,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741935_1115 (size=5958) 2024-11-13T19:54:41,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741935_1115 (size=5958) 2024-11-13T19:54:41,102 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/a/d3cd9a25c7b84242b80368926f060822 2024-11-13T19:54:41,132 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/b/6e4a54cfc45b4804abd8a273cea983e4 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731527676736/Put/seqid=0 2024-11-13T19:54:41,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741936_1116 (size=5958) 2024-11-13T19:54:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741936_1116 (size=5958) 2024-11-13T19:54:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741936_1116 (size=5958) 2024-11-13T19:54:41,150 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/b/6e4a54cfc45b4804abd8a273cea983e4 2024-11-13T19:54:41,200 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/c/bb013c10aba34263b466e138549f9ba2 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731527676745/Put/seqid=0 2024-11-13T19:54:41,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741937_1117 (size=5958) 2024-11-13T19:54:41,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741937_1117 (size=5958) 2024-11-13T19:54:41,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741937_1117 (size=5958) 2024-11-13T19:54:41,213 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/c/bb013c10aba34263b466e138549f9ba2 2024-11-13T19:54:41,223 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/a/d3cd9a25c7b84242b80368926f060822 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/a/d3cd9a25c7b84242b80368926f060822 2024-11-13T19:54:41,232 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/a/d3cd9a25c7b84242b80368926f060822, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T19:54:41,233 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/b/6e4a54cfc45b4804abd8a273cea983e4 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/b/6e4a54cfc45b4804abd8a273cea983e4 2024-11-13T19:54:41,238 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/b/6e4a54cfc45b4804abd8a273cea983e4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T19:54:41,239 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/.tmp/c/bb013c10aba34263b466e138549f9ba2 as hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/c/bb013c10aba34263b466e138549f9ba2 2024-11-13T19:54:41,244 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/c/bb013c10aba34263b466e138549f9ba2, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T19:54:41,244 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 895b71ec5b37de3f74e3ff3a2ca95081 in 188ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T19:54:41,245 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/0000000000000000066 2024-11-13T19:54:41,246 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,246 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,247 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T19:54:41,249 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 895b71ec5b37de3f74e3ff3a2ca95081 2024-11-13T19:54:41,253 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/hbase/data/default/testReplayEditsWrittenViaHRegion/895b71ec5b37de3f74e3ff3a2ca95081/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T19:54:41,254 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 895b71ec5b37de3f74e3ff3a2ca95081; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74527048, jitterRate=0.11053955554962158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T19:54:41,255 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 895b71ec5b37de3f74e3ff3a2ca95081: Writing region info on filesystem at 1731527680990Initializing all the Stores at 1731527680991 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527680991Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527680991Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731527680991Obtaining lock to block concurrent updates at 1731527681056 (+65 ms)Preparing flush snapshotting stores in 895b71ec5b37de3f74e3ff3a2ca95081 at 1731527681056Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731527681056Flushing stores of testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. at 1731527681056Flushing 895b71ec5b37de3f74e3ff3a2ca95081/a: creating writer at 1731527681056Flushing 895b71ec5b37de3f74e3ff3a2ca95081/a: appending metadata at 1731527681078 (+22 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/a: closing flushed file at 1731527681078Flushing 895b71ec5b37de3f74e3ff3a2ca95081/b: creating writer at 1731527681112 (+34 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/b: appending metadata at 1731527681131 (+19 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/b: closing flushed file at 1731527681131Flushing 895b71ec5b37de3f74e3ff3a2ca95081/c: creating writer at 1731527681166 (+35 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/c: appending metadata at 1731527681199 (+33 ms)Flushing 895b71ec5b37de3f74e3ff3a2ca95081/c: closing flushed file at 1731527681200 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2eacb7c7: reopening flushed file at 1731527681223 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dfd7e58: reopening flushed file at 1731527681232 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ff406e0: reopening flushed file at 1731527681239 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 895b71ec5b37de3f74e3ff3a2ca95081 in 188ms, sequenceid=66, compaction requested=false; wal=null at 1731527681244 (+5 ms)Cleaning up temporary data from old regions at 1731527681246 (+2 ms)Region opened successfully at 1731527681255 (+9 ms) 2024-11-13T19:54:41,272 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 895b71ec5b37de3f74e3ff3a2ca95081, disabling compactions & flushes 2024-11-13T19:54:41,273 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:41,273 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:41,273 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. after waiting 0 ms 2024-11-13T19:54:41,273 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:41,278 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731527675799.895b71ec5b37de3f74e3ff3a2ca95081. 2024-11-13T19:54:41,278 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 895b71ec5b37de3f74e3ff3a2ca95081: Waiting for close lock at 1731527681272Disabling compacts and flushes for region at 1731527681272Disabling writes for close at 1731527681273 (+1 ms)Writing region close event to WAL at 1731527681278 (+5 ms)Closed at 1731527681278 2024-11-13T19:54:41,282 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527680935 not finished, retry = 0 2024-11-13T19:54:41,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741934_1114 (size=175) 2024-11-13T19:54:41,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741934_1114 (size=175) 2024-11-13T19:54:41,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741934_1114 (size=175) 2024-11-13T19:54:41,386 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T19:54:41,386 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731527680935) 2024-11-13T19:54:41,402 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=437 (was 434) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:39891 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_987450744_22 at /127.0.0.1:39946 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1323050754) connection to localhost/127.0.0.1:39891 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_987450744_22 at /127.0.0.1:53862 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1402 (was 1340) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=466 (was 481), ProcessCount=11 (was 11), AvailableMemoryMB=2419 (was 2292) - AvailableMemoryMB LEAK? - 2024-11-13T19:54:41,402 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1402 is superior to 1024 2024-11-13T19:54:41,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T19:54:41,402 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:54:41,402 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:54:41,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:41,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:41,403 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:54:41,403 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T19:54:41,403 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=223486682, stopped=false 2024-11-13T19:54:41,404 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=54d0784d1943,35115,1731527634648 2024-11-13T19:54:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T19:54:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T19:54:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:54:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:54:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T19:54:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:54:41,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T19:54:41,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:54:41,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:54:41,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:54:41,409 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:54:41,409 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:54:41,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:41,410 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '54d0784d1943,46741,1731527635545' ***** 2024-11-13T19:54:41,410 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T19:54:41,410 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '54d0784d1943,35943,1731527635712' ***** 2024-11-13T19:54:41,410 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T19:54:41,410 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T19:54:41,410 INFO [RS:2;54d0784d1943:35943 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T19:54:41,410 INFO [RS:2;54d0784d1943:35943 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T19:54:41,410 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(959): stopping server 54d0784d1943,35943,1731527635712 2024-11-13T19:54:41,410 INFO [RS:2;54d0784d1943:35943 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:54:41,410 INFO [RS:2;54d0784d1943:35943 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;54d0784d1943:35943. 2024-11-13T19:54:41,410 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T19:54:41,411 DEBUG [RS:2;54d0784d1943:35943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:54:41,411 DEBUG [RS:2;54d0784d1943:35943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T19:54:41,411 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(976): stopping server 54d0784d1943,35943,1731527635712; all regions closed. 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(3091): Received CLOSE for ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(959): stopping server 54d0784d1943,46741,1731527635545 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;54d0784d1943:46741. 2024-11-13T19:54:41,411 DEBUG [RS:0;54d0784d1943:46741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:54:41,411 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T19:54:41,411 DEBUG [RS:0;54d0784d1943:46741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T19:54:41,411 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T19:54:41,412 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T19:54:41,412 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab2bd533f2e133bf9e32cd3965a8e16d, disabling compactions & flushes 2024-11-13T19:54:41,412 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:41,412 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:41,412 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. after waiting 0 ms 2024-11-13T19:54:41,412 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:41,421 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T19:54:41,421 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ab2bd533f2e133bf9e32cd3965a8e16d=testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d.} 2024-11-13T19:54:41,421 DEBUG [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ab2bd533f2e133bf9e32cd3965a8e16d 2024-11-13T19:54:41,421 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T19:54:41,422 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T19:54:41,422 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T19:54:41,422 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T19:54:41,422 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T19:54:41,422 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-13T19:54:41,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741833_1009 (size=175) 2024-11-13T19:54:41,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741833_1009 (size=175) 2024-11-13T19:54:41,428 INFO [regionserver/54d0784d1943:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:54:41,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741833_1009 (size=175) 2024-11-13T19:54:41,438 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/default/testReplayEditsAfterRegionMovedWithMultiCF/ab2bd533f2e133bf9e32cd3965a8e16d/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T19:54:41,439 INFO [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:41,439 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab2bd533f2e133bf9e32cd3965a8e16d: Waiting for close lock at 1731527681412Running coprocessor pre-close hooks at 1731527681412Disabling compacts and flushes for region at 1731527681412Disabling writes for close at 1731527681412Writing region close event to WAL at 1731527681424 (+12 ms)Running coprocessor post-close hooks at 1731527681439 (+15 ms)Closed at 1731527681439 2024-11-13T19:54:41,439 DEBUG [RS_CLOSE_REGION-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d. 2024-11-13T19:54:41,443 DEBUG [RS:2;54d0784d1943:35943 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 54d0784d1943%2C35943%2C1731527635712:(num 1731527637542) 2024-11-13T19:54:41,443 DEBUG [RS:2;54d0784d1943:35943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] hbase.ChoreService(370): Chore service for: regionserver/54d0784d1943:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T19:54:41,443 INFO [regionserver/54d0784d1943:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:54:41,443 INFO [RS:2;54d0784d1943:35943 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35943 2024-11-13T19:54:41,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/54d0784d1943,35943,1731527635712 2024-11-13T19:54:41,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:54:41,446 INFO [RS:2;54d0784d1943:35943 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:54:41,448 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [54d0784d1943,35943,1731527635712] 2024-11-13T19:54:41,450 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/54d0784d1943,35943,1731527635712 already deleted, retry=false 2024-11-13T19:54:41,450 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 54d0784d1943,35943,1731527635712 expired; onlineServers=1 2024-11-13T19:54:41,466 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/info/8114c5447f1641b0b412473e9adc5b21 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731527654147.ab2bd533f2e133bf9e32cd3965a8e16d./info:regioninfo/1731527657269/Put/seqid=0 2024-11-13T19:54:41,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741938_1118 (size=8243) 2024-11-13T19:54:41,478 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/info/8114c5447f1641b0b412473e9adc5b21 2024-11-13T19:54:41,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741938_1118 (size=8243) 2024-11-13T19:54:41,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741938_1118 (size=8243) 2024-11-13T19:54:41,492 INFO [regionserver/54d0784d1943:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:54:41,504 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/ns/2afba3205c9c473f91d1be77d930d6c3 is 43, key is default/ns:d/1731527638409/Put/seqid=0 2024-11-13T19:54:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741939_1119 (size=5153) 2024-11-13T19:54:41,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741939_1119 (size=5153) 2024-11-13T19:54:41,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741939_1119 (size=5153) 2024-11-13T19:54:41,517 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/ns/2afba3205c9c473f91d1be77d930d6c3 2024-11-13T19:54:41,544 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/table/454abf7e95144129b0561520e18de332 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731527654586/Put/seqid=0 2024-11-13T19:54:41,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:41,549 INFO [RS:2;54d0784d1943:35943 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:54:41,549 INFO [RS:2;54d0784d1943:35943 {}] regionserver.HRegionServer(1031): Exiting; stopping=54d0784d1943,35943,1731527635712; zookeeper connection closed. 2024-11-13T19:54:41,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35943-0x100eda9e0200003, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:41,553 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7323c3b4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7323c3b4 2024-11-13T19:54:41,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741940_1120 (size=5431) 2024-11-13T19:54:41,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741940_1120 (size=5431) 2024-11-13T19:54:41,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741940_1120 (size=5431) 2024-11-13T19:54:41,557 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/table/454abf7e95144129b0561520e18de332 2024-11-13T19:54:41,568 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/info/8114c5447f1641b0b412473e9adc5b21 as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/info/8114c5447f1641b0b412473e9adc5b21 2024-11-13T19:54:41,576 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/info/8114c5447f1641b0b412473e9adc5b21, entries=18, sequenceid=21, filesize=8.0 K 2024-11-13T19:54:41,577 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/ns/2afba3205c9c473f91d1be77d930d6c3 as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/ns/2afba3205c9c473f91d1be77d930d6c3 2024-11-13T19:54:41,583 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/ns/2afba3205c9c473f91d1be77d930d6c3, entries=2, sequenceid=21, filesize=5.0 K 2024-11-13T19:54:41,584 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/.tmp/table/454abf7e95144129b0561520e18de332 as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/table/454abf7e95144129b0561520e18de332 2024-11-13T19:54:41,589 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/table/454abf7e95144129b0561520e18de332, entries=2, sequenceid=21, filesize=5.3 K 2024-11-13T19:54:41,591 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 169ms, sequenceid=21, compaction requested=false 2024-11-13T19:54:41,596 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-13T19:54:41,596 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:54:41,596 INFO [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T19:54:41,597 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731527681421Running coprocessor pre-close hooks at 1731527681421Disabling compacts and flushes for region at 1731527681421Disabling writes for close at 1731527681422 (+1 ms)Obtaining lock to block concurrent updates at 1731527681422Preparing flush snapshotting stores in 1588230740 at 1731527681422Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731527681423 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731527681425 (+2 ms)Flushing 1588230740/info: creating writer at 1731527681427 (+2 ms)Flushing 1588230740/info: appending metadata at 1731527681465 (+38 ms)Flushing 1588230740/info: closing flushed file at 1731527681466 (+1 ms)Flushing 1588230740/ns: creating writer at 1731527681485 (+19 ms)Flushing 1588230740/ns: appending metadata at 1731527681503 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731527681503Flushing 1588230740/table: creating writer at 1731527681523 (+20 ms)Flushing 1588230740/table: appending metadata at 1731527681544 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731527681544Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62743db2: reopening flushed file at 1731527681567 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76d610e: reopening flushed file at 1731527681576 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@697c3c5e: reopening flushed file at 1731527681583 (+7 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 169ms, sequenceid=21, compaction requested=false at 1731527681591 (+8 ms)Writing region close event to WAL at 1731527681592 (+1 ms)Running coprocessor post-close hooks at 1731527681596 (+4 ms)Closed at 1731527681596 2024-11-13T19:54:41,597 DEBUG [RS_CLOSE_META-regionserver/54d0784d1943:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T19:54:41,622 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(976): stopping server 54d0784d1943,46741,1731527635545; all regions closed. 2024-11-13T19:54:41,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741836_1012 (size=9487) 2024-11-13T19:54:41,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741836_1012 (size=9487) 2024-11-13T19:54:41,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741836_1012 (size=9487) 2024-11-13T19:54:41,628 DEBUG [RS:0;54d0784d1943:46741 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs 2024-11-13T19:54:41,628 INFO [RS:0;54d0784d1943:46741 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 54d0784d1943%2C46741%2C1731527635545.meta:.meta(num 1731527638163) 2024-11-13T19:54:41,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741835_1011 (size=3621) 2024-11-13T19:54:41,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741835_1011 (size=3621) 2024-11-13T19:54:41,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741835_1011 (size=3621) 2024-11-13T19:54:41,634 DEBUG [RS:0;54d0784d1943:46741 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/oldWALs 2024-11-13T19:54:41,634 INFO [RS:0;54d0784d1943:46741 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 54d0784d1943%2C46741%2C1731527635545:(num 1731527637542) 2024-11-13T19:54:41,634 DEBUG [RS:0;54d0784d1943:46741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:54:41,634 INFO [RS:0;54d0784d1943:46741 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:54:41,634 INFO [RS:0;54d0784d1943:46741 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:54:41,634 INFO [RS:0;54d0784d1943:46741 {}] hbase.ChoreService(370): Chore service for: regionserver/54d0784d1943:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T19:54:41,634 INFO [RS:0;54d0784d1943:46741 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:54:41,634 INFO [regionserver/54d0784d1943:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:54:41,634 INFO [RS:0;54d0784d1943:46741 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46741 2024-11-13T19:54:41,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:54:41,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/54d0784d1943,46741,1731527635545 2024-11-13T19:54:41,637 INFO [RS:0;54d0784d1943:46741 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:54:41,637 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [54d0784d1943,46741,1731527635545] 2024-11-13T19:54:41,639 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/54d0784d1943,46741,1731527635545 already deleted, retry=false 2024-11-13T19:54:41,639 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 54d0784d1943,46741,1731527635545 expired; onlineServers=0 2024-11-13T19:54:41,639 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '54d0784d1943,35115,1731527634648' ***** 2024-11-13T19:54:41,639 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T19:54:41,639 INFO [M:0;54d0784d1943:35115 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:54:41,639 INFO [M:0;54d0784d1943:35115 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:54:41,640 DEBUG [M:0;54d0784d1943:35115 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T19:54:41,640 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T19:54:41,640 DEBUG [M:0;54d0784d1943:35115 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T19:54:41,640 DEBUG [master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.large.0-1731527637051 {}] cleaner.HFileCleaner(306): Exit Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.large.0-1731527637051,5,FailOnTimeoutGroup] 2024-11-13T19:54:41,640 INFO [M:0;54d0784d1943:35115 {}] hbase.ChoreService(370): Chore service for: master/54d0784d1943:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T19:54:41,640 INFO [M:0;54d0784d1943:35115 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:54:41,640 DEBUG [M:0;54d0784d1943:35115 {}] master.HMaster(1795): Stopping service threads 2024-11-13T19:54:41,640 INFO [M:0;54d0784d1943:35115 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T19:54:41,640 INFO [M:0;54d0784d1943:35115 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T19:54:41,641 DEBUG [master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.small.0-1731527637062 {}] cleaner.HFileCleaner(306): Exit Thread[master/54d0784d1943:0:becomeActiveMaster-HFileCleaner.small.0-1731527637062,5,FailOnTimeoutGroup] 2024-11-13T19:54:41,641 INFO [M:0;54d0784d1943:35115 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T19:54:41,641 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T19:54:41,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T19:54:41,642 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:54:41,642 DEBUG [M:0;54d0784d1943:35115 {}] zookeeper.ZKUtil(347): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T19:54:41,642 WARN [M:0;54d0784d1943:35115 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T19:54:41,643 INFO [M:0;54d0784d1943:35115 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/.lastflushedseqids 2024-11-13T19:54:41,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741941_1121 (size=138) 2024-11-13T19:54:41,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741941_1121 (size=138) 2024-11-13T19:54:41,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741941_1121 (size=138) 2024-11-13T19:54:41,660 INFO [M:0;54d0784d1943:35115 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T19:54:41,661 INFO [M:0;54d0784d1943:35115 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T19:54:41,661 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T19:54:41,661 INFO [M:0;54d0784d1943:35115 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:54:41,661 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:54:41,661 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T19:54:41,661 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:54:41,661 INFO [M:0;54d0784d1943:35115 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-13T19:54:41,677 DEBUG [M:0;54d0784d1943:35115 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/81d5483991c942db8b1fc064716382ba is 82, key is hbase:meta,,1/info:regioninfo/1731527638316/Put/seqid=0 2024-11-13T19:54:41,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741942_1122 (size=5672) 2024-11-13T19:54:41,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741942_1122 (size=5672) 2024-11-13T19:54:41,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741942_1122 (size=5672) 2024-11-13T19:54:41,684 INFO [M:0;54d0784d1943:35115 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/81d5483991c942db8b1fc064716382ba 2024-11-13T19:54:41,704 DEBUG [M:0;54d0784d1943:35115 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fead15ab05754585b57ed128f6ae4388 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731527654591/Put/seqid=0 2024-11-13T19:54:41,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741943_1123 (size=7755) 2024-11-13T19:54:41,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741943_1123 (size=7755) 2024-11-13T19:54:41,714 INFO [M:0;54d0784d1943:35115 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fead15ab05754585b57ed128f6ae4388 2024-11-13T19:54:41,721 INFO [M:0;54d0784d1943:35115 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fead15ab05754585b57ed128f6ae4388 2024-11-13T19:54:41,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741943_1123 (size=7755) 2024-11-13T19:54:41,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:41,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46741-0x100eda9e0200001, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:41,743 INFO [RS:0;54d0784d1943:46741 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:54:41,743 INFO [RS:0;54d0784d1943:46741 {}] regionserver.HRegionServer(1031): Exiting; stopping=54d0784d1943,46741,1731527635545; zookeeper connection closed. 2024-11-13T19:54:41,745 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5575cd3a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5575cd3a 2024-11-13T19:54:41,745 DEBUG [M:0;54d0784d1943:35115 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/411408e1e1bf46878e7379041edcbccc is 69, key is 54d0784d1943,35943,1731527635712/rs:state/1731527637185/Put/seqid=0 2024-11-13T19:54:41,745 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T19:54:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741944_1124 (size=5445) 2024-11-13T19:54:41,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741944_1124 (size=5445) 2024-11-13T19:54:41,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741944_1124 (size=5445) 2024-11-13T19:54:41,770 INFO [M:0;54d0784d1943:35115 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/411408e1e1bf46878e7379041edcbccc 2024-11-13T19:54:41,775 INFO [M:0;54d0784d1943:35115 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 411408e1e1bf46878e7379041edcbccc 2024-11-13T19:54:41,776 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/81d5483991c942db8b1fc064716382ba as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/81d5483991c942db8b1fc064716382ba 2024-11-13T19:54:41,782 INFO [M:0;54d0784d1943:35115 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/81d5483991c942db8b1fc064716382ba, entries=8, sequenceid=168, filesize=5.5 K 2024-11-13T19:54:41,789 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fead15ab05754585b57ed128f6ae4388 as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fead15ab05754585b57ed128f6ae4388 2024-11-13T19:54:41,800 INFO [M:0;54d0784d1943:35115 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fead15ab05754585b57ed128f6ae4388 2024-11-13T19:54:41,800 INFO [M:0;54d0784d1943:35115 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fead15ab05754585b57ed128f6ae4388, entries=17, sequenceid=168, filesize=7.6 K 2024-11-13T19:54:41,804 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/411408e1e1bf46878e7379041edcbccc as hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/411408e1e1bf46878e7379041edcbccc 2024-11-13T19:54:41,817 INFO [M:0;54d0784d1943:35115 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 411408e1e1bf46878e7379041edcbccc 2024-11-13T19:54:41,817 INFO [M:0;54d0784d1943:35115 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39891/user/jenkins/test-data/1246db0a-24ec-dca6-f161-fc2096597976/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/411408e1e1bf46878e7379041edcbccc, entries=3, sequenceid=168, filesize=5.3 K 2024-11-13T19:54:41,822 INFO [M:0;54d0784d1943:35115 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 160ms, sequenceid=168, compaction requested=false 2024-11-13T19:54:41,830 INFO [M:0;54d0784d1943:35115 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:54:41,830 DEBUG [M:0;54d0784d1943:35115 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731527681661Disabling compacts and flushes for region at 1731527681661Disabling writes for close at 1731527681661Obtaining lock to block concurrent updates at 1731527681661Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731527681661Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731527681661Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731527681662 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731527681662Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731527681677 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731527681677Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731527681689 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731527681703 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731527681703Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731527681721 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731527681744 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731527681744Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45dd0eb8: reopening flushed file at 1731527681775 (+31 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25f18831: reopening flushed file at 1731527681783 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21863d94: reopening flushed file at 1731527681801 (+18 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 160ms, sequenceid=168, compaction requested=false at 1731527681822 (+21 ms)Writing region close event to WAL at 1731527681830 (+8 ms)Closed at 1731527681830 2024-11-13T19:54:41,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33527 is added to blk_1073741830_1006 (size=83606) 2024-11-13T19:54:41,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33541 is added to blk_1073741830_1006 (size=83606) 2024-11-13T19:54:41,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43261 is added to blk_1073741830_1006 (size=83606) 2024-11-13T19:54:41,839 INFO [M:0;54d0784d1943:35115 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T19:54:41,839 INFO [M:0;54d0784d1943:35115 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35115 2024-11-13T19:54:41,840 INFO [M:0;54d0784d1943:35115 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:54:41,840 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:54:41,953 INFO [M:0;54d0784d1943:35115 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:54:41,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:41,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35115-0x100eda9e0200000, quorum=127.0.0.1:57594, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:54:41,960 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527658925 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527658925 (inode 16655) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731527657625/wal.1731527658925 (inode 16655) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T19:54:41,961 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731527648839/wal.1731527649119 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:41,961 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731527675798/wal.1731527676561 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:41,963 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527669416 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527669416 (inode 16678) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731527659298/wal.1731527669416 (inode 16678) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T19:54:41,963 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731527640771/wal.1731527640883 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:41,966 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527670581 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527670581 (inode 16704) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731527669724/wal.1731527670581 (inode 16704) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T19:54:41,966 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731527670902/wal.1731527671085 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:54:41,968 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal.1731527640556 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal.1731527640556 (inode 16485) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731527640054/wal.1731527640556 (inode 16485) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T19:54:41,971 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal.1731527639735 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal.1731527639735 (inode 16462) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731527638705/wal.1731527639735 (inode 16462) Holder DFSClient_NONMAPREDUCE_-228963406_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T19:54:41,981 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5deffb5d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:54:41,984 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5a646eb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:54:41,984 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:54:41,984 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7161498b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:54:41,984 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16c07ee3{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,STOPPED} 2024-11-13T19:54:41,987 WARN [BP-1458969915-172.17.0.2-1731527631214 heartbeating to localhost/127.0.0.1:39891 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T19:54:41,987 WARN [BP-1458969915-172.17.0.2-1731527631214 heartbeating to localhost/127.0.0.1:39891 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1458969915-172.17.0.2-1731527631214 (Datanode Uuid 962dc963-8913-476b-9d54-664e01726004) service to localhost/127.0.0.1:39891 2024-11-13T19:54:41,987 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T19:54:41,988 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T19:54:41,989 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data5/current/BP-1458969915-172.17.0.2-1731527631214 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:54:41,990 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data6/current/BP-1458969915-172.17.0.2-1731527631214 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:54:41,990 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T19:54:41,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5705c253{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:54:41,994 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5af9d354{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:54:41,994 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:54:41,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b034666{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:54:41,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f9065a5{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,STOPPED} 2024-11-13T19:54:41,996 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T19:54:41,996 WARN [BP-1458969915-172.17.0.2-1731527631214 heartbeating to localhost/127.0.0.1:39891 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T19:54:41,996 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T19:54:41,996 WARN [BP-1458969915-172.17.0.2-1731527631214 heartbeating to localhost/127.0.0.1:39891 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1458969915-172.17.0.2-1731527631214 (Datanode Uuid 3b4f5e91-0f84-4e4e-b294-53a707cf967f) service to localhost/127.0.0.1:39891 2024-11-13T19:54:41,997 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data3/current/BP-1458969915-172.17.0.2-1731527631214 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:54:41,997 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data4/current/BP-1458969915-172.17.0.2-1731527631214 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:54:41,998 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T19:54:42,000 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1be71666{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:54:42,000 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@9edd34e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:54:42,000 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:54:42,000 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cf706e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:54:42,000 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fcc4f36{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,STOPPED} 2024-11-13T19:54:42,001 WARN [BP-1458969915-172.17.0.2-1731527631214 heartbeating to localhost/127.0.0.1:39891 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T19:54:42,001 WARN [BP-1458969915-172.17.0.2-1731527631214 heartbeating to localhost/127.0.0.1:39891 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1458969915-172.17.0.2-1731527631214 (Datanode Uuid 6b11cc4b-8d1d-403a-a10f-829b720687bb) service to localhost/127.0.0.1:39891 2024-11-13T19:54:42,002 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data2/current/BP-1458969915-172.17.0.2-1731527631214 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:54:42,002 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T19:54:42,005 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/cluster_8ab29dab-5bdc-ba07-b59c-c4591f8370a4/data/data1/current/BP-1458969915-172.17.0.2-1731527631214 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:54:42,005 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T19:54:42,006 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T19:54:42,009 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5608f426{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T19:54:42,010 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:54:42,010 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:54:42,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:54:42,010 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/326d289d-d457-fb7f-42af-3ce58021acd1/hadoop.log.dir/,STOPPED} 2024-11-13T19:54:42,023 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T19:54:42,078 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down