2024-11-24 04:50:18,695 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 04:50:18,708 main DEBUG Took 0.011437 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 04:50:18,709 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 04:50:18,709 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 04:50:18,710 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 04:50:18,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,718 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 04:50:18,729 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,731 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,731 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,731 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,733 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,733 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,733 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,734 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,735 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,736 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,737 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,738 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,739 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 04:50:18,739 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,740 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 04:50:18,741 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 04:50:18,742 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 04:50:18,744 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 04:50:18,744 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 04:50:18,745 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 04:50:18,746 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 04:50:18,753 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 04:50:18,755 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 04:50:18,757 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 04:50:18,757 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 04:50:18,758 main DEBUG createAppenders(={Console}) 2024-11-24 04:50:18,758 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 04:50:18,759 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 04:50:18,759 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 04:50:18,759 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 04:50:18,760 main DEBUG OutputStream closed 2024-11-24 04:50:18,760 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 04:50:18,760 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 04:50:18,760 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 04:50:18,827 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 04:50:18,829 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 04:50:18,830 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 04:50:18,831 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 04:50:18,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 04:50:18,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 04:50:18,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 04:50:18,832 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 04:50:18,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 04:50:18,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 04:50:18,833 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 04:50:18,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 04:50:18,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 04:50:18,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 04:50:18,834 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 04:50:18,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 04:50:18,835 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 04:50:18,836 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 04:50:18,838 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 04:50:18,838 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 04:50:18,838 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 04:50:18,839 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T04:50:19,051 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933 2024-11-24 04:50:19,054 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 04:50:19,054 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T04:50:19,062 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T04:50:19,068 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayCompressed timeout: 13 mins 2024-11-24T04:50:19,088 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T04:50:19,135 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T04:50:19,136 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T04:50:19,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T04:50:19,164 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217, deleteOnExit=true 2024-11-24T04:50:19,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T04:50:19,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/test.cache.data in system properties and HBase conf 2024-11-24T04:50:19,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T04:50:19,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir in system properties and HBase conf 2024-11-24T04:50:19,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T04:50:19,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T04:50:19,168 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T04:50:19,250 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T04:50:19,358 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T04:50:19,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:50:19,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T04:50:19,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T04:50:19,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:50:19,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T04:50:19,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T04:50:19,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T04:50:19,365 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:50:19,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T04:50:19,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/nfs.dump.dir in system properties and HBase conf 2024-11-24T04:50:19,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/java.io.tmpdir in system properties and HBase conf 2024-11-24T04:50:19,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T04:50:19,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T04:50:19,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T04:50:20,432 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T04:50:20,495 INFO [Time-limited test {}] log.Log(170): Logging initialized @2482ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T04:50:20,560 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:50:20,620 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:50:20,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:50:20,640 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:50:20,641 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:50:20,657 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:50:20,660 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:50:20,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:50:20,823 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1843526c{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/java.io.tmpdir/jetty-localhost-39071-hadoop-hdfs-3_4_1-tests_jar-_-any-3579622644443514053/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:50:20,829 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:39071} 2024-11-24T04:50:20,829 INFO [Time-limited test {}] server.Server(415): Started @2816ms 2024-11-24T04:50:21,505 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:50:21,512 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:50:21,513 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:50:21,513 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:50:21,514 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T04:50:21,514 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:50:21,515 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:50:21,613 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4dc44ec6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/java.io.tmpdir/jetty-localhost-40107-hadoop-hdfs-3_4_1-tests_jar-_-any-17239354409322606948/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:50:21,614 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:40107} 2024-11-24T04:50:21,614 INFO [Time-limited test {}] server.Server(415): Started @3602ms 2024-11-24T04:50:21,659 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:50:21,763 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:50:21,768 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:50:21,769 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:50:21,769 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:50:21,769 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:50:21,772 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:50:21,772 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:50:21,871 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28e1ba78{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/java.io.tmpdir/jetty-localhost-46191-hadoop-hdfs-3_4_1-tests_jar-_-any-4488643723358316256/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:50:21,872 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:46191} 2024-11-24T04:50:21,872 INFO [Time-limited test {}] server.Server(415): Started @3860ms 2024-11-24T04:50:21,874 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:50:21,907 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T04:50:21,912 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T04:50:21,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T04:50:21,913 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T04:50:21,913 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T04:50:21,914 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,AVAILABLE} 2024-11-24T04:50:21,915 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T04:50:22,051 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4290616c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/java.io.tmpdir/jetty-localhost-45153-hadoop-hdfs-3_4_1-tests_jar-_-any-9748537645614597008/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:50:22,052 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:45153} 2024-11-24T04:50:22,052 INFO [Time-limited test {}] server.Server(415): Started @4039ms 2024-11-24T04:50:22,054 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T04:50:23,755 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data4/current/BP-1774581998-172.17.0.2-1732423819869/current, will proceed with Du for space computation calculation, 2024-11-24T04:50:23,755 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data2/current/BP-1774581998-172.17.0.2-1732423819869/current, will proceed with Du for space computation calculation, 2024-11-24T04:50:23,755 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data3/current/BP-1774581998-172.17.0.2-1732423819869/current, will proceed with Du for space computation calculation, 2024-11-24T04:50:23,755 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data1/current/BP-1774581998-172.17.0.2-1732423819869/current, will proceed with Du for space computation calculation, 2024-11-24T04:50:23,785 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:50:23,785 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:50:23,831 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7333786454e14c20 with lease ID 0xce70e3021723974e: Processing first storage report for DS-5b7a04c0-5eec-4454-824e-302d129fa12a from datanode DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f93db99e-cdc0-4f99-a3b0-7770d06b2821, infoPort=38959, infoSecurePort=0, ipcPort=37543, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869) 2024-11-24T04:50:23,832 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7333786454e14c20 with lease ID 0xce70e3021723974e: from storage DS-5b7a04c0-5eec-4454-824e-302d129fa12a node DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f93db99e-cdc0-4f99-a3b0-7770d06b2821, infoPort=38959, infoSecurePort=0, ipcPort=37543, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T04:50:23,833 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8f15d816d5791977 with lease ID 0xce70e3021723974f: Processing first storage report for DS-1802503b-9d87-44a3-86e5-5385b2a93562 from datanode DatanodeRegistration(127.0.0.1:43517, datanodeUuid=19d587c1-17c9-409b-ac27-73e2c9f40c5d, infoPort=41661, infoSecurePort=0, ipcPort=46655, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869) 2024-11-24T04:50:23,833 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8f15d816d5791977 with lease ID 0xce70e3021723974f: from storage DS-1802503b-9d87-44a3-86e5-5385b2a93562 node DatanodeRegistration(127.0.0.1:43517, datanodeUuid=19d587c1-17c9-409b-ac27-73e2c9f40c5d, infoPort=41661, infoSecurePort=0, ipcPort=46655, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:50:23,833 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7333786454e14c20 with lease ID 0xce70e3021723974e: Processing first storage report for DS-bd8306d2-facd-4f3d-b2a5-0a7fada6e856 from datanode DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f93db99e-cdc0-4f99-a3b0-7770d06b2821, infoPort=38959, infoSecurePort=0, ipcPort=37543, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869) 2024-11-24T04:50:23,833 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7333786454e14c20 with lease ID 0xce70e3021723974e: from storage DS-bd8306d2-facd-4f3d-b2a5-0a7fada6e856 node DatanodeRegistration(127.0.0.1:46055, datanodeUuid=f93db99e-cdc0-4f99-a3b0-7770d06b2821, infoPort=38959, infoSecurePort=0, ipcPort=37543, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:50:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8f15d816d5791977 with lease ID 0xce70e3021723974f: Processing first storage report for DS-252be1ec-cb3d-482d-be0e-d0107cc5deb8 from datanode DatanodeRegistration(127.0.0.1:43517, datanodeUuid=19d587c1-17c9-409b-ac27-73e2c9f40c5d, infoPort=41661, infoSecurePort=0, ipcPort=46655, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869) 2024-11-24T04:50:23,834 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8f15d816d5791977 with lease ID 0xce70e3021723974f: from storage DS-252be1ec-cb3d-482d-be0e-d0107cc5deb8 node DatanodeRegistration(127.0.0.1:43517, datanodeUuid=19d587c1-17c9-409b-ac27-73e2c9f40c5d, infoPort=41661, infoSecurePort=0, ipcPort=46655, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:50:23,847 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data5/current/BP-1774581998-172.17.0.2-1732423819869/current, will proceed with Du for space computation calculation, 2024-11-24T04:50:23,847 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data6/current/BP-1774581998-172.17.0.2-1732423819869/current, will proceed with Du for space computation calculation, 2024-11-24T04:50:23,869 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T04:50:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6b2f8fd03136715 with lease ID 0xce70e30217239750: Processing first storage report for DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7 from datanode DatanodeRegistration(127.0.0.1:42305, datanodeUuid=024efb5a-2ac3-457e-9045-db84e38ed4a0, infoPort=44567, infoSecurePort=0, ipcPort=40927, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869) 2024-11-24T04:50:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6b2f8fd03136715 with lease ID 0xce70e30217239750: from storage DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7 node DatanodeRegistration(127.0.0.1:42305, datanodeUuid=024efb5a-2ac3-457e-9045-db84e38ed4a0, infoPort=44567, infoSecurePort=0, ipcPort=40927, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:50:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x6b2f8fd03136715 with lease ID 0xce70e30217239750: Processing first storage report for DS-f9d79b7b-6bf7-4282-89fb-7d0b2190f978 from datanode DatanodeRegistration(127.0.0.1:42305, datanodeUuid=024efb5a-2ac3-457e-9045-db84e38ed4a0, infoPort=44567, infoSecurePort=0, ipcPort=40927, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869) 2024-11-24T04:50:23,874 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x6b2f8fd03136715 with lease ID 0xce70e30217239750: from storage DS-f9d79b7b-6bf7-4282-89fb-7d0b2190f978 node DatanodeRegistration(127.0.0.1:42305, datanodeUuid=024efb5a-2ac3-457e-9045-db84e38ed4a0, infoPort=44567, infoSecurePort=0, ipcPort=40927, storageInfo=lv=-57;cid=testClusterID;nsid=1020842275;c=1732423819869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T04:50:23,891 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933 2024-11-24T04:50:23,956 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/zookeeper_0, clientPort=49287, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T04:50:23,965 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49287 2024-11-24T04:50:23,991 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:23,995 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:24,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:50:24,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:50:24,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741825_1001 (size=7) 2024-11-24T04:50:24,604 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322 with version=8 2024-11-24T04:50:24,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/hbase-staging 2024-11-24T04:50:24,874 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:50:24,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:24,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:24,888 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:50:24,888 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:24,888 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:50:25,012 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T04:50:25,065 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T04:50:25,074 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T04:50:25,078 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:50:25,102 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 56362 (auto-detected) 2024-11-24T04:50:25,103 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T04:50:25,123 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36681 2024-11-24T04:50:25,143 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36681 connecting to ZooKeeper ensemble=127.0.0.1:49287 2024-11-24T04:50:25,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:366810x0, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:50:25,265 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36681-0x1016b2dadec0000 connected 2024-11-24T04:50:25,377 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,381 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:50:25,396 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322, hbase.cluster.distributed=false 2024-11-24T04:50:25,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:50:25,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36681 2024-11-24T04:50:25,422 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36681 2024-11-24T04:50:25,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36681 2024-11-24T04:50:25,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36681 2024-11-24T04:50:25,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36681 2024-11-24T04:50:25,519 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:50:25,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,520 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:50:25,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:50:25,523 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:50:25,526 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:50:25,527 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36869 2024-11-24T04:50:25,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36869 connecting to ZooKeeper ensemble=127.0.0.1:49287 2024-11-24T04:50:25,530 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,534 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368690x0, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:50:25,553 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:368690x0, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:50:25,554 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36869-0x1016b2dadec0001 connected 2024-11-24T04:50:25,557 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:50:25,564 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:50:25,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:50:25,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:50:25,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36869 2024-11-24T04:50:25,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36869 2024-11-24T04:50:25,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36869 2024-11-24T04:50:25,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36869 2024-11-24T04:50:25,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36869 2024-11-24T04:50:25,594 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:50:25,595 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,595 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,595 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:50:25,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,596 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:50:25,596 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:50:25,596 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:50:25,597 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46377 2024-11-24T04:50:25,599 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46377 connecting to ZooKeeper ensemble=127.0.0.1:49287 2024-11-24T04:50:25,600 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,603 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,615 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:463770x0, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:50:25,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:463770x0, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:50:25,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46377-0x1016b2dadec0002 connected 2024-11-24T04:50:25,617 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:50:25,618 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:50:25,619 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:50:25,621 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:50:25,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46377 2024-11-24T04:50:25,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46377 2024-11-24T04:50:25,626 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46377 2024-11-24T04:50:25,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46377 2024-11-24T04:50:25,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46377 2024-11-24T04:50:25,646 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4464c5b832df:0 server-side Connection retries=45 2024-11-24T04:50:25,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,647 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T04:50:25,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T04:50:25,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T04:50:25,647 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T04:50:25,647 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T04:50:25,648 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42181 2024-11-24T04:50:25,650 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42181 connecting to ZooKeeper ensemble=127.0.0.1:49287 2024-11-24T04:50:25,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,654 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421810x0, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T04:50:25,669 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421810x0, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:50:25,669 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42181-0x1016b2dadec0003 connected 2024-11-24T04:50:25,670 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T04:50:25,670 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T04:50:25,671 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T04:50:25,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T04:50:25,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42181 2024-11-24T04:50:25,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42181 2024-11-24T04:50:25,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42181 2024-11-24T04:50:25,680 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42181 2024-11-24T04:50:25,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42181 2024-11-24T04:50:25,695 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4464c5b832df:36681 2024-11-24T04:50:25,695 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4464c5b832df,36681,1732423824726 2024-11-24T04:50:25,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,713 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4464c5b832df,36681,1732423824726 2024-11-24T04:50:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:50:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:50:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T04:50:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,743 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T04:50:25,744 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4464c5b832df,36681,1732423824726 from backup master directory 2024-11-24T04:50:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4464c5b832df,36681,1732423824726 2024-11-24T04:50:25,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T04:50:25,753 WARN [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:50:25,753 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4464c5b832df,36681,1732423824726 2024-11-24T04:50:25,756 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T04:50:25,757 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T04:50:25,810 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/hbase.id] with ID: 287bef7b-d135-4e75-8479-3603de413d72 2024-11-24T04:50:25,810 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/.tmp/hbase.id 2024-11-24T04:50:25,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:50:25,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:50:25,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741826_1002 (size=42) 2024-11-24T04:50:25,826 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/.tmp/hbase.id]:[hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/hbase.id] 2024-11-24T04:50:25,873 INFO [master/4464c5b832df:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T04:50:25,878 INFO [master/4464c5b832df:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T04:50:25,897 INFO [master/4464c5b832df:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-24T04:50:25,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:25,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:50:25,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:50:25,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741827_1003 (size=196) 2024-11-24T04:50:25,949 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:50:25,950 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T04:50:25,955 INFO [master/4464c5b832df:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:25,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:50:25,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:50:25,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741828_1004 (size=1189) 2024-11-24T04:50:26,001 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store 2024-11-24T04:50:26,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:50:26,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:50:26,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741829_1005 (size=34) 2024-11-24T04:50:26,026 INFO [master/4464c5b832df:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T04:50:26,029 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:26,030 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:50:26,030 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:50:26,031 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:50:26,032 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:50:26,032 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:50:26,032 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:50:26,034 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732423826030Disabling compacts and flushes for region at 1732423826030Disabling writes for close at 1732423826032 (+2 ms)Writing region close event to WAL at 1732423826032Closed at 1732423826032 2024-11-24T04:50:26,036 WARN [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/.initializing 2024-11-24T04:50:26,036 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/WALs/4464c5b832df,36681,1732423824726 2024-11-24T04:50:26,044 INFO [master/4464c5b832df:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:26,059 INFO [master/4464c5b832df:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C36681%2C1732423824726, suffix=, logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/WALs/4464c5b832df,36681,1732423824726, archiveDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/oldWALs, maxLogs=10 2024-11-24T04:50:26,094 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/WALs/4464c5b832df,36681,1732423824726/4464c5b832df%2C36681%2C1732423824726.1732423826064, exclude list is [], retry=0 2024-11-24T04:50:26,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:26,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:26,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:26,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:26,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T04:50:26,155 INFO [master/4464c5b832df:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/WALs/4464c5b832df,36681,1732423824726/4464c5b832df%2C36681%2C1732423824726.1732423826064 2024-11-24T04:50:26,156 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:26,156 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:26,156 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:26,159 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,160 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,195 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T04:50:26,221 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:26,224 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T04:50:26,228 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:26,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T04:50:26,232 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:26,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,237 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T04:50:26,237 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,238 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:26,238 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,242 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,243 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,248 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,249 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,252 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:50:26,256 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T04:50:26,261 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:26,263 INFO [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70395439, jitterRate=0.04897378385066986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:50:26,271 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732423826171Initializing all the Stores at 1732423826173 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423826174 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423826174Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423826174Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423826175 (+1 ms)Cleaning up temporary data from old regions at 1732423826249 (+74 ms)Region opened successfully at 1732423826271 (+22 ms) 2024-11-24T04:50:26,273 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T04:50:26,307 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2256a475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:50:26,336 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T04:50:26,345 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T04:50:26,345 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T04:50:26,348 INFO [master/4464c5b832df:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T04:50:26,349 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T04:50:26,353 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-24T04:50:26,354 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T04:50:26,376 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T04:50:26,383 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T04:50:26,394 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T04:50:26,396 INFO [master/4464c5b832df:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T04:50:26,398 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T04:50:26,404 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T04:50:26,407 INFO [master/4464c5b832df:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T04:50:26,411 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T04:50:26,422 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T04:50:26,424 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T04:50:26,436 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T04:50:26,458 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T04:50:26,467 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T04:50:26,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:50:26,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:50:26,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:50:26,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T04:50:26,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,483 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4464c5b832df,36681,1732423824726, sessionid=0x1016b2dadec0000, setting cluster-up flag (Was=false) 2024-11-24T04:50:26,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,542 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T04:50:26,547 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4464c5b832df,36681,1732423824726 2024-11-24T04:50:26,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:26,605 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T04:50:26,608 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4464c5b832df,36681,1732423824726 2024-11-24T04:50:26,618 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T04:50:26,685 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(746): ClusterId : 287bef7b-d135-4e75-8479-3603de413d72 2024-11-24T04:50:26,685 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(746): ClusterId : 287bef7b-d135-4e75-8479-3603de413d72 2024-11-24T04:50:26,685 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(746): ClusterId : 287bef7b-d135-4e75-8479-3603de413d72 2024-11-24T04:50:26,687 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:50:26,687 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:50:26,687 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T04:50:26,689 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T04:50:26,698 INFO [master/4464c5b832df:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T04:50:26,701 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:50:26,701 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:50:26,701 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T04:50:26,701 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:50:26,701 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:50:26,701 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T04:50:26,705 INFO [master/4464c5b832df:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T04:50:26,710 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4464c5b832df,36681,1732423824726 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T04:50:26,721 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:50:26,721 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:50:26,721 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T04:50:26,722 DEBUG [RS:1;4464c5b832df:46377 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670c9898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:50:26,722 DEBUG [RS:2;4464c5b832df:42181 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61985ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:50:26,722 DEBUG [RS:0;4464c5b832df:36869 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fbe39b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4464c5b832df/172.17.0.2:0 2024-11-24T04:50:26,723 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:50:26,723 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:50:26,723 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:50:26,723 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4464c5b832df:0, corePoolSize=5, maxPoolSize=5 2024-11-24T04:50:26,723 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4464c5b832df:0, corePoolSize=10, maxPoolSize=10 2024-11-24T04:50:26,723 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,724 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:50:26,724 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,730 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732423856730 2024-11-24T04:50:26,733 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T04:50:26,733 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:50:26,733 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T04:50:26,734 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T04:50:26,738 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T04:50:26,739 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T04:50:26,739 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T04:50:26,740 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T04:50:26,741 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,742 DEBUG [RS:1;4464c5b832df:46377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4464c5b832df:46377 2024-11-24T04:50:26,742 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4464c5b832df:36869 2024-11-24T04:50:26,742 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;4464c5b832df:42181 2024-11-24T04:50:26,742 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T04:50:26,742 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,747 INFO [RS:0;4464c5b832df:36869 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:50:26,747 INFO [RS:1;4464c5b832df:46377 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:50:26,747 INFO [RS:2;4464c5b832df:42181 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T04:50:26,747 INFO [RS:1;4464c5b832df:46377 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:50:26,747 INFO [RS:0;4464c5b832df:36869 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:50:26,747 INFO [RS:2;4464c5b832df:42181 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T04:50:26,747 DEBUG [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:50:26,747 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:50:26,747 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T04:50:26,748 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T04:50:26,750 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,36681,1732423824726 with port=36869, startcode=1732423825485 2024-11-24T04:50:26,750 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,36681,1732423824726 with port=46377, startcode=1732423825594 2024-11-24T04:50:26,750 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(2659): reportForDuty to master=4464c5b832df,36681,1732423824726 with port=42181, startcode=1732423825646 2024-11-24T04:50:26,750 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T04:50:26,750 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T04:50:26,756 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T04:50:26,756 INFO [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T04:50:26,758 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.large.0-1732423826758,5,FailOnTimeoutGroup] 2024-11-24T04:50:26,759 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.small.0-1732423826759,5,FailOnTimeoutGroup] 2024-11-24T04:50:26,759 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,760 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T04:50:26,761 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,761 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,762 DEBUG [RS:1;4464c5b832df:46377 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:50:26,762 DEBUG [RS:2;4464c5b832df:42181 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:50:26,762 DEBUG [RS:0;4464c5b832df:36869 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T04:50:26,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:50:26,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:50:26,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741831_1007 (size=1321) 2024-11-24T04:50:26,771 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T04:50:26,771 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322 2024-11-24T04:50:26,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:50:26,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:50:26,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741832_1008 (size=32) 2024-11-24T04:50:26,804 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:26,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:50:26,808 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:50:26,808 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37539, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:50:26,808 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35793, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T04:50:26,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:50:26,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:26,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:50:26,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:50:26,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,816 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36681 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4464c5b832df,42181,1732423825646 2024-11-24T04:50:26,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:26,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:50:26,819 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36681 {}] master.ServerManager(517): Registering regionserver=4464c5b832df,42181,1732423825646 2024-11-24T04:50:26,821 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:50:26,821 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:26,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:50:26,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:50:26,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:26,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:26,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:50:26,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740 2024-11-24T04:50:26,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36681 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4464c5b832df,36869,1732423825485 2024-11-24T04:50:26,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36681 {}] master.ServerManager(517): Registering regionserver=4464c5b832df,36869,1732423825485 2024-11-24T04:50:26,835 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740 2024-11-24T04:50:26,840 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322 2024-11-24T04:50:26,840 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44683 2024-11-24T04:50:26,840 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:50:26,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:50:26,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:50:26,842 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36681 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4464c5b832df,46377,1732423825594 2024-11-24T04:50:26,843 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:50:26,843 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36681 {}] master.ServerManager(517): Registering regionserver=4464c5b832df,46377,1732423825594 2024-11-24T04:50:26,843 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322 2024-11-24T04:50:26,843 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44683 2024-11-24T04:50:26,843 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:50:26,847 DEBUG [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322 2024-11-24T04:50:26,847 DEBUG [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44683 2024-11-24T04:50:26,848 DEBUG [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T04:50:26,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:50:26,854 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:26,855 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68981758, jitterRate=0.027908295392990112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:50:26,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732423826805Initializing all the Stores at 1732423826807 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423826808 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423826808Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423826808Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423826808Cleaning up temporary data from old regions at 1732423826841 (+33 ms)Region opened successfully at 1732423826861 (+20 ms) 2024-11-24T04:50:26,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:50:26,861 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:50:26,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:50:26,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:50:26,862 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:50:26,864 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:50:26,864 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732423826861Disabling compacts and flushes for region at 1732423826861Disabling writes for close at 1732423826862 (+1 ms)Writing region close event to WAL at 1732423826863 (+1 ms)Closed at 1732423826864 (+1 ms) 2024-11-24T04:50:26,868 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:50:26,868 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T04:50:26,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T04:50:26,888 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:50:26,893 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T04:50:26,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:50:26,931 DEBUG [RS:2;4464c5b832df:42181 {}] zookeeper.ZKUtil(111): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4464c5b832df,42181,1732423825646 2024-11-24T04:50:26,931 DEBUG [RS:0;4464c5b832df:36869 {}] zookeeper.ZKUtil(111): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4464c5b832df,36869,1732423825485 2024-11-24T04:50:26,931 WARN [RS:2;4464c5b832df:42181 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:50:26,931 WARN [RS:0;4464c5b832df:36869 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:50:26,931 INFO [RS:0;4464c5b832df:36869 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:26,931 INFO [RS:2;4464c5b832df:42181 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:26,932 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,42181,1732423825646 2024-11-24T04:50:26,932 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485 2024-11-24T04:50:26,932 DEBUG [RS:1;4464c5b832df:46377 {}] zookeeper.ZKUtil(111): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4464c5b832df,46377,1732423825594 2024-11-24T04:50:26,932 WARN [RS:1;4464c5b832df:46377 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T04:50:26,932 INFO [RS:1;4464c5b832df:46377 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:26,932 DEBUG [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,46377,1732423825594 2024-11-24T04:50:26,933 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4464c5b832df,36869,1732423825485] 2024-11-24T04:50:26,933 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4464c5b832df,42181,1732423825646] 2024-11-24T04:50:26,933 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4464c5b832df,46377,1732423825594] 2024-11-24T04:50:26,957 INFO [RS:2;4464c5b832df:42181 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:50:26,957 INFO [RS:0;4464c5b832df:36869 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:50:26,957 INFO [RS:1;4464c5b832df:46377 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T04:50:26,974 INFO [RS:2;4464c5b832df:42181 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:50:26,974 INFO [RS:1;4464c5b832df:46377 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:50:26,978 INFO [RS:0;4464c5b832df:36869 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T04:50:26,982 INFO [RS:0;4464c5b832df:36869 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:50:26,983 INFO [RS:1;4464c5b832df:46377 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:50:26,983 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,983 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,983 INFO [RS:2;4464c5b832df:42181 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:50:26,983 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,984 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:50:26,988 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:50:26,988 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T04:50:26,991 INFO [RS:1;4464c5b832df:46377 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:50:26,991 INFO [RS:2;4464c5b832df:42181 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:50:26,991 INFO [RS:0;4464c5b832df:36869 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T04:50:26,992 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,992 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,992 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:26,993 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,993 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:50:26,994 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0, corePoolSize=2, maxPoolSize=2 2024-11-24T04:50:26,994 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,994 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:50:26,995 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:50:26,995 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,995 DEBUG [RS:0;4464c5b832df:36869 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:50:26,995 DEBUG [RS:2;4464c5b832df:42181 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:50:26,995 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4464c5b832df:0, corePoolSize=1, maxPoolSize=1 2024-11-24T04:50:26,995 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:50:26,995 DEBUG [RS:1;4464c5b832df:46377 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4464c5b832df:0, corePoolSize=3, maxPoolSize=3 2024-11-24T04:50:27,001 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,001 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,002 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,002 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,002 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,002 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,42181,1732423825646-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36869,1732423825485-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:50:27,004 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,005 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,005 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,46377,1732423825594-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:50:27,027 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:50:27,027 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:50:27,029 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,46377,1732423825594-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,029 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36869,1732423825485-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,029 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,029 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,029 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T04:50:27,029 INFO [RS:1;4464c5b832df:46377 {}] regionserver.Replication(171): 4464c5b832df,46377,1732423825594 started 2024-11-24T04:50:27,029 INFO [RS:0;4464c5b832df:36869 {}] regionserver.Replication(171): 4464c5b832df,36869,1732423825485 started 2024-11-24T04:50:27,029 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,42181,1732423825646-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,030 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,030 INFO [RS:2;4464c5b832df:42181 {}] regionserver.Replication(171): 4464c5b832df,42181,1732423825646 started 2024-11-24T04:50:27,044 WARN [4464c5b832df:36681 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T04:50:27,048 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,048 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,049 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1482): Serving as 4464c5b832df,36869,1732423825485, RpcServer on 4464c5b832df/172.17.0.2:36869, sessionid=0x1016b2dadec0001 2024-11-24T04:50:27,049 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(1482): Serving as 4464c5b832df,46377,1732423825594, RpcServer on 4464c5b832df/172.17.0.2:46377, sessionid=0x1016b2dadec0002 2024-11-24T04:50:27,049 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:50:27,049 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:50:27,050 DEBUG [RS:1;4464c5b832df:46377 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4464c5b832df,46377,1732423825594 2024-11-24T04:50:27,050 DEBUG [RS:0;4464c5b832df:36869 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4464c5b832df,36869,1732423825485 2024-11-24T04:50:27,050 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,46377,1732423825594' 2024-11-24T04:50:27,050 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,36869,1732423825485' 2024-11-24T04:50:27,050 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:50:27,050 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:50:27,051 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:50:27,051 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:50:27,052 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:50:27,052 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:50:27,052 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:50:27,052 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:50:27,052 DEBUG [RS:1;4464c5b832df:46377 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4464c5b832df,46377,1732423825594 2024-11-24T04:50:27,052 DEBUG [RS:0;4464c5b832df:36869 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4464c5b832df,36869,1732423825485 2024-11-24T04:50:27,052 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,46377,1732423825594' 2024-11-24T04:50:27,052 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:50:27,053 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,053 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1482): Serving as 4464c5b832df,42181,1732423825646, RpcServer on 4464c5b832df/172.17.0.2:42181, sessionid=0x1016b2dadec0003 2024-11-24T04:50:27,054 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T04:50:27,054 DEBUG [RS:2;4464c5b832df:42181 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4464c5b832df,42181,1732423825646 2024-11-24T04:50:27,054 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:50:27,054 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,42181,1732423825646' 2024-11-24T04:50:27,054 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T04:50:27,054 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,36869,1732423825485' 2024-11-24T04:50:27,055 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:50:27,055 DEBUG [RS:1;4464c5b832df:46377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:50:27,055 INFO [RS:1;4464c5b832df:46377 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:50:27,055 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T04:50:27,055 INFO [RS:1;4464c5b832df:46377 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:50:27,055 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:50:27,056 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T04:50:27,056 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T04:50:27,056 DEBUG [RS:2;4464c5b832df:42181 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4464c5b832df,42181,1732423825646 2024-11-24T04:50:27,056 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4464c5b832df,42181,1732423825646' 2024-11-24T04:50:27,056 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T04:50:27,056 DEBUG [RS:0;4464c5b832df:36869 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:50:27,056 INFO [RS:0;4464c5b832df:36869 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:50:27,056 INFO [RS:0;4464c5b832df:36869 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:50:27,057 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T04:50:27,057 DEBUG [RS:2;4464c5b832df:42181 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T04:50:27,057 INFO [RS:2;4464c5b832df:42181 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T04:50:27,057 INFO [RS:2;4464c5b832df:42181 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T04:50:27,160 INFO [RS:1;4464c5b832df:46377 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:27,160 INFO [RS:0;4464c5b832df:36869 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:27,160 INFO [RS:2;4464c5b832df:42181 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:27,163 INFO [RS:1;4464c5b832df:46377 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C46377%2C1732423825594, suffix=, logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,46377,1732423825594, archiveDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs, maxLogs=32 2024-11-24T04:50:27,163 INFO [RS:2;4464c5b832df:42181 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C42181%2C1732423825646, suffix=, logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,42181,1732423825646, archiveDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs, maxLogs=32 2024-11-24T04:50:27,163 INFO [RS:0;4464c5b832df:36869 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C36869%2C1732423825485, suffix=, logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485, archiveDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs, maxLogs=32 2024-11-24T04:50:27,181 DEBUG [RS:1;4464c5b832df:46377 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,46377,1732423825594/4464c5b832df%2C46377%2C1732423825594.1732423827167, exclude list is [], retry=0 2024-11-24T04:50:27,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:27,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:27,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:27,187 DEBUG [RS:0;4464c5b832df:36869 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485/4464c5b832df%2C36869%2C1732423825485.1732423827167, exclude list is [], retry=0 2024-11-24T04:50:27,187 DEBUG [RS:2;4464c5b832df:42181 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,42181,1732423825646/4464c5b832df%2C42181%2C1732423825646.1732423827167, exclude list is [], retry=0 2024-11-24T04:50:27,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:27,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:27,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:27,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:27,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:27,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:27,196 INFO [RS:1;4464c5b832df:46377 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,46377,1732423825594/4464c5b832df%2C46377%2C1732423825594.1732423827167 2024-11-24T04:50:27,197 DEBUG [RS:1;4464c5b832df:46377 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:27,202 INFO [RS:2;4464c5b832df:42181 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,42181,1732423825646/4464c5b832df%2C42181%2C1732423825646.1732423827167 2024-11-24T04:50:27,202 DEBUG [RS:2;4464c5b832df:42181 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:27,203 INFO [RS:0;4464c5b832df:36869 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485/4464c5b832df%2C36869%2C1732423825485.1732423827167 2024-11-24T04:50:27,204 DEBUG [RS:0;4464c5b832df:36869 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:27,297 DEBUG [4464c5b832df:36681 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T04:50:27,307 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(204): Hosts are {4464c5b832df=0} racks are {/default-rack=0} 2024-11-24T04:50:27,317 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:50:27,317 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:50:27,317 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T04:50:27,317 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:50:27,317 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:50:27,317 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T04:50:27,317 INFO [4464c5b832df:36681 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:50:27,317 INFO [4464c5b832df:36681 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:50:27,317 INFO [4464c5b832df:36681 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T04:50:27,318 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:50:27,327 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4464c5b832df,42181,1732423825646 2024-11-24T04:50:27,333 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4464c5b832df,42181,1732423825646, state=OPENING 2024-11-24T04:50:27,391 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T04:50:27,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:27,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:27,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:27,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:50:27,402 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,403 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,405 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T04:50:27,407 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4464c5b832df,42181,1732423825646}] 2024-11-24T04:50:27,584 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:50:27,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35755, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:50:27,598 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T04:50:27,599 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:27,599 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T04:50:27,602 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4464c5b832df%2C42181%2C1732423825646.meta, suffix=.meta, logDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,42181,1732423825646, archiveDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs, maxLogs=32 2024-11-24T04:50:27,618 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,42181,1732423825646/4464c5b832df%2C42181%2C1732423825646.meta.1732423827604.meta, exclude list is [], retry=0 2024-11-24T04:50:27,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:27,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:27,622 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:27,625 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,42181,1732423825646/4464c5b832df%2C42181%2C1732423825646.meta.1732423827604.meta 2024-11-24T04:50:27,625 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:27,626 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:27,627 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T04:50:27,630 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T04:50:27,635 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T04:50:27,639 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T04:50:27,640 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:27,640 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T04:50:27,640 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T04:50:27,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T04:50:27,644 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T04:50:27,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:27,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:27,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T04:50:27,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T04:50:27,647 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:27,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:27,648 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T04:50:27,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T04:50:27,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:27,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:27,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T04:50:27,652 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T04:50:27,652 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:27,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T04:50:27,653 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T04:50:27,654 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740 2024-11-24T04:50:27,657 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740 2024-11-24T04:50:27,659 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T04:50:27,659 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T04:50:27,660 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T04:50:27,663 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T04:50:27,665 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68297605, jitterRate=0.017713621258735657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T04:50:27,665 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T04:50:27,667 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732423827640Writing region info on filesystem at 1732423827641 (+1 ms)Initializing all the Stores at 1732423827642 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423827642Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423827643 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423827643Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732423827643Cleaning up temporary data from old regions at 1732423827659 (+16 ms)Running coprocessor post-open hooks at 1732423827665 (+6 ms)Region opened successfully at 1732423827667 (+2 ms) 2024-11-24T04:50:27,674 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732423827576 2024-11-24T04:50:27,683 DEBUG [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T04:50:27,684 INFO [RS_OPEN_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T04:50:27,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4464c5b832df,42181,1732423825646 2024-11-24T04:50:27,688 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4464c5b832df,42181,1732423825646, state=OPEN 2024-11-24T04:50:27,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:50:27,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:50:27,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:50:27,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T04:50:27,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T04:50:27,774 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4464c5b832df,42181,1732423825646 2024-11-24T04:50:27,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T04:50:27,779 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4464c5b832df,42181,1732423825646 in 367 msec 2024-11-24T04:50:27,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T04:50:27,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 904 msec 2024-11-24T04:50:27,788 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T04:50:27,788 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T04:50:27,805 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:50:27,806 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4464c5b832df,42181,1732423825646, seqNum=-1] 2024-11-24T04:50:27,824 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:50:27,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39173, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:50:27,846 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1980 sec 2024-11-24T04:50:27,846 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732423827846, completionTime=-1 2024-11-24T04:50:27,849 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T04:50:27,849 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T04:50:27,874 INFO [master/4464c5b832df:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T04:50:27,874 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732423887874 2024-11-24T04:50:27,874 INFO [master/4464c5b832df:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732423947874 2024-11-24T04:50:27,874 INFO [master/4464c5b832df:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-24T04:50:27,876 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T04:50:27,883 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36681,1732423824726-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,883 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36681,1732423824726-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,883 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36681,1732423824726-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,884 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4464c5b832df:36681, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,885 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,885 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,914 DEBUG [master/4464c5b832df:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T04:50:27,933 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.180sec 2024-11-24T04:50:27,935 INFO [master/4464c5b832df:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T04:50:27,936 INFO [master/4464c5b832df:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T04:50:27,937 INFO [master/4464c5b832df:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T04:50:27,937 INFO [master/4464c5b832df:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T04:50:27,938 INFO [master/4464c5b832df:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T04:50:27,938 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36681,1732423824726-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T04:50:27,939 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36681,1732423824726-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T04:50:27,948 DEBUG [master/4464c5b832df:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T04:50:27,948 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T04:50:27,949 INFO [master/4464c5b832df:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4464c5b832df,36681,1732423824726-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:27,994 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4136470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:50:27,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4464c5b832df,36681,-1 for getting cluster id 2024-11-24T04:50:27,998 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T04:50:28,007 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '287bef7b-d135-4e75-8479-3603de413d72' 2024-11-24T04:50:28,010 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T04:50:28,010 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "287bef7b-d135-4e75-8479-3603de413d72" 2024-11-24T04:50:28,013 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1837f710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:50:28,013 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4464c5b832df,36681,-1] 2024-11-24T04:50:28,016 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T04:50:28,018 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:50:28,019 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T04:50:28,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f3e6a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T04:50:28,023 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T04:50:28,032 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4464c5b832df,42181,1732423825646, seqNum=-1] 2024-11-24T04:50:28,033 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:50:28,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:50:28,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4464c5b832df,36681,1732423824726 2024-11-24T04:50:28,056 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44683/hbase 2024-11-24T04:50:28,075 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=366, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=11278 2024-11-24T04:50:28,100 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:28,103 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:28,104 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:28,109 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-46301206, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-46301206, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:28,124 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-46301206/hregion-46301206.1732423828110, exclude list is [], retry=0 2024-11-24T04:50:28,128 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:28,128 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:28,128 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:28,133 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-46301206/hregion-46301206.1732423828110 2024-11-24T04:50:28,134 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:28,134 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1a407009be6258f4be63686501fb7bb0, NAME => 'testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:28,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:50:28,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:50:28,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741838_1014 (size=64) 2024-11-24T04:50:28,151 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:28,153 INFO [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,156 INFO [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a407009be6258f4be63686501fb7bb0 columnFamilyName a 2024-11-24T04:50:28,156 DEBUG [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:28,157 INFO [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] regionserver.HStore(327): Store=1a407009be6258f4be63686501fb7bb0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:28,158 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,159 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,160 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,161 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,161 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,164 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,167 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:28,168 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a407009be6258f4be63686501fb7bb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72982645, jitterRate=0.08752615749835968}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:28,169 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a407009be6258f4be63686501fb7bb0: Writing region info on filesystem at 1732423828151Initializing all the Stores at 1732423828153 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423828153Cleaning up temporary data from old regions at 1732423828161 (+8 ms)Region opened successfully at 1732423828169 (+8 ms) 2024-11-24T04:50:28,169 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1a407009be6258f4be63686501fb7bb0, disabling compactions & flushes 2024-11-24T04:50:28,169 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0. 2024-11-24T04:50:28,169 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0. 2024-11-24T04:50:28,169 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0. after waiting 0 ms 2024-11-24T04:50:28,169 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0. 2024-11-24T04:50:28,170 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0. 2024-11-24T04:50:28,170 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1a407009be6258f4be63686501fb7bb0: Waiting for close lock at 1732423828169Disabling compacts and flushes for region at 1732423828169Disabling writes for close at 1732423828169Writing region close event to WAL at 1732423828170 (+1 ms)Closed at 1732423828170 2024-11-24T04:50:28,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741837_1013 (size=93) 2024-11-24T04:50:28,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741837_1013 (size=93) 2024-11-24T04:50:28,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741837_1013 (size=93) 2024-11-24T04:50:28,183 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:28,183 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-46301206:(num 1732423828110) 2024-11-24T04:50:28,185 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:50:28,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741839_1015 (size=276) 2024-11-24T04:50:28,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741839_1015 (size=276) 2024-11-24T04:50:28,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741839_1015 (size=276) 2024-11-24T04:50:28,204 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:50:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741840_1016 (size=230) 2024-11-24T04:50:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741840_1016 (size=230) 2024-11-24T04:50:28,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741840_1016 (size=230) 2024-11-24T04:50:28,245 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1, size=276 (276bytes) 2024-11-24T04:50:28,246 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T04:50:28,246 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T04:50:28,246 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1 2024-11-24T04:50:28,252 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1 after 4ms 2024-11-24T04:50:28,258 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,259 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1 took 16ms 2024-11-24T04:50:28,263 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1 so closing down 2024-11-24T04:50:28,267 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T04:50:28,268 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T04:50:28,269 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:28,270 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:28,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741841_1017 (size=276) 2024-11-24T04:50:28,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741841_1017 (size=276) 2024-11-24T04:50:28,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741841_1017 (size=276) 2024-11-24T04:50:28,282 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:28,284 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002 2024-11-24T04:50:28,287 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-24T04:50:28,288 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1, journal: Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1, size=276 (276bytes) at 1732423828245Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1 so closing down at 1732423828263 (+18 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000001-wal-1.temp at 1732423828269 (+6 ms)3 split writer threads finished at 1732423828270 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732423828282 (+12 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002 at 1732423828284 (+2 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732423828287 (+3 ms) 2024-11-24T04:50:28,302 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2, size=230 (230bytes) 2024-11-24T04:50:28,302 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2 2024-11-24T04:50:28,303 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2 after 1ms 2024-11-24T04:50:28,307 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,308 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2 took 6ms 2024-11-24T04:50:28,310 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2 so closing down 2024-11-24T04:50:28,310 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:28,312 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T04:50:28,314 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T04:50:28,314 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:28,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741842_1018 (size=230) 2024-11-24T04:50:28,324 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:28,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741842_1018 (size=230) 2024-11-24T04:50:28,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741842_1018 (size=230) 2024-11-24T04:50:28,330 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,332 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002-wal-2.temp, length=230 2024-11-24T04:50:28,334 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-24T04:50:28,334 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2, journal: Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2, size=230 (230bytes) at 1732423828302Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2 so closing down at 1732423828310 (+8 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002-wal-2.temp at 1732423828314 (+4 ms)3 split writer threads finished at 1732423828314Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732423828324 (+10 ms)Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732423828334 (+10 ms) 2024-11-24T04:50:28,334 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:28,337 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:28,350 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal.1732423828338, exclude list is [], retry=0 2024-11-24T04:50:28,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:28,355 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:28,356 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:28,358 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal.1732423828338 2024-11-24T04:50:28,359 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a407009be6258f4be63686501fb7bb0, NAME => 'testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,362 INFO [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,363 INFO [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a407009be6258f4be63686501fb7bb0 columnFamilyName a 2024-11-24T04:50:28,363 DEBUG [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:28,365 INFO [StoreOpener-1a407009be6258f4be63686501fb7bb0-1 {}] regionserver.HStore(327): Store=1a407009be6258f4be63686501fb7bb0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:28,365 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,370 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,371 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002 2024-11-24T04:50:28,374 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,382 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002 2024-11-24T04:50:28,386 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a407009be6258f4be63686501fb7bb0 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T04:50:28,446 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/.tmp/a/40ccd73f35354eb4beabb8f3cc31dabd is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732423828183/Put/seqid=0 2024-11-24T04:50:28,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:50:28,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:50:28,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741844_1020 (size=5170) 2024-11-24T04:50:28,461 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/.tmp/a/40ccd73f35354eb4beabb8f3cc31dabd 2024-11-24T04:50:28,510 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/.tmp/a/40ccd73f35354eb4beabb8f3cc31dabd as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/a/40ccd73f35354eb4beabb8f3cc31dabd 2024-11-24T04:50:28,522 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/a/40ccd73f35354eb4beabb8f3cc31dabd, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T04:50:28,527 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1a407009be6258f4be63686501fb7bb0 in 141ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T04:50:28,529 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/0000000000000000002 2024-11-24T04:50:28,530 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,530 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,534 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1a407009be6258f4be63686501fb7bb0 2024-11-24T04:50:28,537 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/1a407009be6258f4be63686501fb7bb0/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T04:50:28,539 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1a407009be6258f4be63686501fb7bb0; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69037633, jitterRate=0.02874089777469635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:28,541 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1a407009be6258f4be63686501fb7bb0: Writing region info on filesystem at 1732423828360Initializing all the Stores at 1732423828361 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423828361Obtaining lock to block concurrent updates at 1732423828386 (+25 ms)Preparing flush snapshotting stores in 1a407009be6258f4be63686501fb7bb0 at 1732423828386Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732423828390 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732423828101.1a407009be6258f4be63686501fb7bb0. at 1732423828390Flushing 1a407009be6258f4be63686501fb7bb0/a: creating writer at 1732423828392 (+2 ms)Flushing 1a407009be6258f4be63686501fb7bb0/a: appending metadata at 1732423828435 (+43 ms)Flushing 1a407009be6258f4be63686501fb7bb0/a: closing flushed file at 1732423828437 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e384f8d: reopening flushed file at 1732423828508 (+71 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 1a407009be6258f4be63686501fb7bb0 in 141ms, sequenceid=2, compaction requested=false; wal=null at 1732423828527 (+19 ms)Cleaning up temporary data from old regions at 1732423828530 (+3 ms)Region opened successfully at 1732423828540 (+10 ms) 2024-11-24T04:50:28,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit0 Thread=378 (was 366) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43858 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51972 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:57026 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:56868 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51962 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44683/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43770 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 456), ProcessCount=11 (was 11), AvailableMemoryMB=11247 (was 11278) 2024-11-24T04:50:28,583 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=378, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=11247 2024-11-24T04:50:28,597 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:28,599 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:28,600 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:28,604 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-10007037, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-10007037, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:28,617 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-10007037/hregion-10007037.1732423828604, exclude list is [], retry=0 2024-11-24T04:50:28,621 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:28,621 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:28,621 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:28,625 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-10007037/hregion-10007037.1732423828604 2024-11-24T04:50:28,625 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:28,626 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ec02c0f10823585ad96fc31ddc103eaa, NAME => 'testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:28,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:50:28,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:50:28,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741846_1022 (size=64) 2024-11-24T04:50:28,643 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:28,644 INFO [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,646 INFO [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec02c0f10823585ad96fc31ddc103eaa columnFamilyName a 2024-11-24T04:50:28,646 DEBUG [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:28,647 INFO [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] regionserver.HStore(327): Store=ec02c0f10823585ad96fc31ddc103eaa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:28,647 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,649 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,649 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,651 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,654 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:28,655 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec02c0f10823585ad96fc31ddc103eaa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70712296, jitterRate=0.05369532108306885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:28,655 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec02c0f10823585ad96fc31ddc103eaa: Writing region info on filesystem at 1732423828643Initializing all the Stores at 1732423828644 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423828644Cleaning up temporary data from old regions at 1732423828649 (+5 ms)Region opened successfully at 1732423828655 (+6 ms) 2024-11-24T04:50:28,655 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ec02c0f10823585ad96fc31ddc103eaa, disabling compactions & flushes 2024-11-24T04:50:28,655 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa. 2024-11-24T04:50:28,655 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa. 2024-11-24T04:50:28,655 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa. after waiting 0 ms 2024-11-24T04:50:28,655 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa. 2024-11-24T04:50:28,656 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa. 2024-11-24T04:50:28,656 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ec02c0f10823585ad96fc31ddc103eaa: Waiting for close lock at 1732423828655Disabling compacts and flushes for region at 1732423828655Disabling writes for close at 1732423828655Writing region close event to WAL at 1732423828656 (+1 ms)Closed at 1732423828656 2024-11-24T04:50:28,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741845_1021 (size=93) 2024-11-24T04:50:28,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741845_1021 (size=93) 2024-11-24T04:50:28,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741845_1021 (size=93) 2024-11-24T04:50:28,666 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:28,666 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-10007037:(num 1732423828604) 2024-11-24T04:50:28,667 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:50:28,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741847_1023 (size=276) 2024-11-24T04:50:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741847_1023 (size=276) 2024-11-24T04:50:28,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741847_1023 (size=276) 2024-11-24T04:50:28,692 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T04:50:28,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741848_1024 (size=230) 2024-11-24T04:50:28,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741848_1024 (size=230) 2024-11-24T04:50:28,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741848_1024 (size=230) 2024-11-24T04:50:28,721 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2, size=230 (230bytes) 2024-11-24T04:50:28,721 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2 2024-11-24T04:50:28,722 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2 after 1ms 2024-11-24T04:50:28,725 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,725 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2 took 5ms 2024-11-24T04:50:28,728 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2 so closing down 2024-11-24T04:50:28,728 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:28,730 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T04:50:28,732 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T04:50:28,732 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:28,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741849_1025 (size=230) 2024-11-24T04:50:28,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741849_1025 (size=230) 2024-11-24T04:50:28,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741849_1025 (size=230) 2024-11-24T04:50:28,744 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:28,746 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002 2024-11-24T04:50:28,746 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2, size=230, length=230, corrupted=false, cancelled=false 2024-11-24T04:50:28,746 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2, journal: Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2, size=230 (230bytes) at 1732423828721Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2 so closing down at 1732423828728 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002-wal-2.temp at 1732423828732 (+4 ms)3 split writer threads finished at 1732423828732Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732423828744 (+12 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002 at 1732423828746 (+2 ms)Processed 1 edits across 1 Regions in 21 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-2, size=230, length=230, corrupted=false, cancelled=false at 1732423828746 2024-11-24T04:50:28,761 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1, size=276 (276bytes) 2024-11-24T04:50:28,761 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1 2024-11-24T04:50:28,762 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1 after 1ms 2024-11-24T04:50:28,765 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,765 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1 took 4ms 2024-11-24T04:50:28,768 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1 so closing down 2024-11-24T04:50:28,768 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:28,771 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T04:50:28,772 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T04:50:28,773 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:28,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741850_1026 (size=276) 2024-11-24T04:50:28,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741850_1026 (size=276) 2024-11-24T04:50:28,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741850_1026 (size=276) 2024-11-24T04:50:28,785 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:28,796 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,799 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002, length=230 2024-11-24T04:50:28,801 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002 2024-11-24T04:50:28,801 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1, size=276, length=276, corrupted=false, cancelled=false 2024-11-24T04:50:28,801 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1, journal: Splitting hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1, size=276 (276bytes) at 1732423828761Finishing writing output for hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1 so closing down at 1732423828768 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000001-wal-1.temp at 1732423828773 (+5 ms)3 split writer threads finished at 1732423828773Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732423828785 (+12 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002 at 1732423828801 (+16 ms)Processed 2 edits across 1 Regions in 35 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal-1, size=276, length=276, corrupted=false, cancelled=false at 1732423828801 2024-11-24T04:50:28,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:28,804 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:28,819 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal.1732423828805, exclude list is [], retry=0 2024-11-24T04:50:28,822 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:28,823 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:28,823 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:28,825 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal.1732423828805 2024-11-24T04:50:28,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:28,825 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ec02c0f10823585ad96fc31ddc103eaa, NAME => 'testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:28,826 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:28,826 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,826 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,827 INFO [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,829 INFO [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec02c0f10823585ad96fc31ddc103eaa columnFamilyName a 2024-11-24T04:50:28,829 DEBUG [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:28,830 INFO [StoreOpener-ec02c0f10823585ad96fc31ddc103eaa-1 {}] regionserver.HStore(327): Store=ec02c0f10823585ad96fc31ddc103eaa/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:28,830 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,831 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,833 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,834 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002 2024-11-24T04:50:28,837 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:28,838 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002 2024-11-24T04:50:28,838 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec02c0f10823585ad96fc31ddc103eaa 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T04:50:28,854 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/.tmp/a/dc56d45632b84a8698ffc432a4a4860b is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732423828666/Put/seqid=0 2024-11-24T04:50:28,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:50:28,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:50:28,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741852_1028 (size=5170) 2024-11-24T04:50:28,863 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/.tmp/a/dc56d45632b84a8698ffc432a4a4860b 2024-11-24T04:50:28,873 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/.tmp/a/dc56d45632b84a8698ffc432a4a4860b as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/a/dc56d45632b84a8698ffc432a4a4860b 2024-11-24T04:50:28,882 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/a/dc56d45632b84a8698ffc432a4a4860b, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T04:50:28,882 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ec02c0f10823585ad96fc31ddc103eaa in 44ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T04:50:28,883 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/0000000000000000002 2024-11-24T04:50:28,884 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,884 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,887 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec02c0f10823585ad96fc31ddc103eaa 2024-11-24T04:50:28,890 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/ec02c0f10823585ad96fc31ddc103eaa/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T04:50:28,891 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec02c0f10823585ad96fc31ddc103eaa; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71932182, jitterRate=0.07187303900718689}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:28,892 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec02c0f10823585ad96fc31ddc103eaa: Writing region info on filesystem at 1732423828826Initializing all the Stores at 1732423828827 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423828827Obtaining lock to block concurrent updates at 1732423828838 (+11 ms)Preparing flush snapshotting stores in ec02c0f10823585ad96fc31ddc103eaa at 1732423828838Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732423828839 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732423828598.ec02c0f10823585ad96fc31ddc103eaa. at 1732423828839Flushing ec02c0f10823585ad96fc31ddc103eaa/a: creating writer at 1732423828839Flushing ec02c0f10823585ad96fc31ddc103eaa/a: appending metadata at 1732423828853 (+14 ms)Flushing ec02c0f10823585ad96fc31ddc103eaa/a: closing flushed file at 1732423828853Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d3248e6: reopening flushed file at 1732423828871 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ec02c0f10823585ad96fc31ddc103eaa in 44ms, sequenceid=2, compaction requested=false; wal=null at 1732423828882 (+11 ms)Cleaning up temporary data from old regions at 1732423828884 (+2 ms)Region opened successfully at 1732423828891 (+7 ms) 2024-11-24T04:50:28,908 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testNameConflictWhenSplit1 Thread=388 (was 378) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:52072 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:56868 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43930 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51962 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:57092 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43770 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 456), ProcessCount=11 (was 11), AvailableMemoryMB=11240 (was 11247) 2024-11-24T04:50:28,919 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=388, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=11239 2024-11-24T04:50:28,933 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:28,935 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:28,936 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:28,939 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82946365, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-82946365, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:28,952 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82946365/hregion-82946365.1732423828940, exclude list is [], retry=0 2024-11-24T04:50:28,956 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:28,956 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:28,956 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:28,962 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82946365/hregion-82946365.1732423828940 2024-11-24T04:50:28,962 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:28,962 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 367677cb81527582e9c34df2dbe01a2e, NAME => 'testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:28,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:50:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:50:28,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741854_1030 (size=64) 2024-11-24T04:50:29,376 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:29,378 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,380 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367677cb81527582e9c34df2dbe01a2e columnFamilyName a 2024-11-24T04:50:29,380 DEBUG [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:29,380 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(327): Store=367677cb81527582e9c34df2dbe01a2e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:29,381 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,382 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367677cb81527582e9c34df2dbe01a2e columnFamilyName b 2024-11-24T04:50:29,382 DEBUG [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:29,383 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(327): Store=367677cb81527582e9c34df2dbe01a2e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:29,383 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,385 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367677cb81527582e9c34df2dbe01a2e columnFamilyName c 2024-11-24T04:50:29,385 DEBUG [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:29,386 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(327): Store=367677cb81527582e9c34df2dbe01a2e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:29,386 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,387 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,388 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,389 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,389 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,390 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:29,391 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:29,394 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:29,394 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 367677cb81527582e9c34df2dbe01a2e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74931464, jitterRate=0.11656582355499268}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:29,395 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 367677cb81527582e9c34df2dbe01a2e: Writing region info on filesystem at 1732423829376Initializing all the Stores at 1732423829377 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423829377Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423829377Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423829377Cleaning up temporary data from old regions at 1732423829389 (+12 ms)Region opened successfully at 1732423829395 (+6 ms) 2024-11-24T04:50:29,395 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 367677cb81527582e9c34df2dbe01a2e, disabling compactions & flushes 2024-11-24T04:50:29,395 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:29,395 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:29,395 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. after waiting 0 ms 2024-11-24T04:50:29,395 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:29,396 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:29,396 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 367677cb81527582e9c34df2dbe01a2e: Waiting for close lock at 1732423829395Disabling compacts and flushes for region at 1732423829395Disabling writes for close at 1732423829395Writing region close event to WAL at 1732423829396 (+1 ms)Closed at 1732423829396 2024-11-24T04:50:29,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741853_1029 (size=93) 2024-11-24T04:50:29,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741853_1029 (size=93) 2024-11-24T04:50:29,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741853_1029 (size=93) 2024-11-24T04:50:29,403 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:29,403 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82946365:(num 1732423828940) 2024-11-24T04:50:29,403 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:29,405 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:29,418 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406, exclude list is [], retry=0 2024-11-24T04:50:29,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:29,422 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:29,423 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:29,425 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 2024-11-24T04:50:29,429 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:29,628 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406, size=0 (0bytes) 2024-11-24T04:50:29,628 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 might be still open, length is 0 2024-11-24T04:50:29,628 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 2024-11-24T04:50:29,629 WARN [IPC Server handler 2 on default port 44683 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T04:50:29,630 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 after 2ms 2024-11-24T04:50:29,891 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43948 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43948 dst: /127.0.0.1:43517 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43517 remote=/127.0.0.1:43948]. Total timeout mills is 60000, 59703 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:29,892 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:57124 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46055:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57124 dst: /127.0.0.1:46055 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:29,892 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:52110 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52110 dst: /127.0.0.1:42305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:29,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741855_1032 (size=200589) 2024-11-24T04:50:29,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741855_1032 (size=200589) 2024-11-24T04:50:29,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741855_1032 (size=200589) 2024-11-24T04:50:33,150 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:50:33,204 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T04:50:33,631 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 after 4003ms 2024-11-24T04:50:33,639 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:33,641 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 took 4013ms 2024-11-24T04:50:33,646 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732423829406.temp 2024-11-24T04:50:33,651 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000000001-wal.1732423829406.temp 2024-11-24T04:50:33,759 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406; continuing. 2024-11-24T04:50:33,759 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 so closing down 2024-11-24T04:50:33,759 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:33,759 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:33,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741856_1033 (size=200597) 2024-11-24T04:50:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741856_1033 (size=200597) 2024-11-24T04:50:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741856_1033 (size=200597) 2024-11-24T04:50:33,766 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000000001-wal.1732423829406.temp (wrote 3002 edits, skipped 0 edits in 48 ms) 2024-11-24T04:50:33,767 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000000001-wal.1732423829406.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000003002 2024-11-24T04:50:33,768 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:50:33,768 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406, journal: Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406, size=0 (0bytes) at 1732423829628Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000000001-wal.1732423829406.temp at 1732423833651 (+4023 ms)Split 1024 edits, skipped 0 edits. at 1732423833706 (+55 ms)Split 2048 edits, skipped 0 edits. at 1732423833736 (+30 ms)Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 so closing down at 1732423833759 (+23 ms)3 split writer threads finished at 1732423833759Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000000001-wal.1732423829406.temp (wrote 3002 edits, skipped 0 edits in 48 ms) at 1732423833766 (+7 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000000001-wal.1732423829406.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000003002 at 1732423833767 (+1 ms)Processed 3002 edits across 1 Regions in 125 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406, size=0, length=0, corrupted=false, cancelled=false at 1732423833768 (+1 ms) 2024-11-24T04:50:33,770 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423829406 2024-11-24T04:50:33,771 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000003002 2024-11-24T04:50:33,772 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:33,775 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:33,791 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423833775, exclude list is [], retry=0 2024-11-24T04:50:33,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:33,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:33,796 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:33,800 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423833775 2024-11-24T04:50:33,803 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:33,804 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:33,807 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:33,808 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367677cb81527582e9c34df2dbe01a2e columnFamilyName a 2024-11-24T04:50:33,808 DEBUG [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:33,809 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(327): Store=367677cb81527582e9c34df2dbe01a2e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:33,809 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:33,811 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367677cb81527582e9c34df2dbe01a2e columnFamilyName b 2024-11-24T04:50:33,811 DEBUG [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:33,812 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(327): Store=367677cb81527582e9c34df2dbe01a2e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:33,812 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:33,813 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 367677cb81527582e9c34df2dbe01a2e columnFamilyName c 2024-11-24T04:50:33,813 DEBUG [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:33,814 INFO [StoreOpener-367677cb81527582e9c34df2dbe01a2e-1 {}] regionserver.HStore(327): Store=367677cb81527582e9c34df2dbe01a2e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:33,814 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:33,816 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:33,819 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:33,820 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000003002 2024-11-24T04:50:33,824 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:33,881 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:50:34,252 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 367677cb81527582e9c34df2dbe01a2e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:50:34,297 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/a/2dd63f35c6c24daaa4ad5dcba317a33a is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732423829435/Put/seqid=0 2024-11-24T04:50:34,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:50:34,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:50:34,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741858_1035 (size=50463) 2024-11-24T04:50:34,309 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/a/2dd63f35c6c24daaa4ad5dcba317a33a 2024-11-24T04:50:34,318 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/a/2dd63f35c6c24daaa4ad5dcba317a33a as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/a/2dd63f35c6c24daaa4ad5dcba317a33a 2024-11-24T04:50:34,326 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/a/2dd63f35c6c24daaa4ad5dcba317a33a, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T04:50:34,327 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 367677cb81527582e9c34df2dbe01a2e in 75ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T04:50:34,357 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:50:34,357 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 367677cb81527582e9c34df2dbe01a2e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:50:34,367 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/a/c44ba3471cdf4aa4841a4993c98678e9 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732423829475/Put/seqid=0 2024-11-24T04:50:34,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:50:34,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:50:34,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741859_1036 (size=20072) 2024-11-24T04:50:34,377 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/a/c44ba3471cdf4aa4841a4993c98678e9 2024-11-24T04:50:34,404 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/b/06c6b68efe684734ba9132558dd63768 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732423829498/Put/seqid=0 2024-11-24T04:50:34,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:50:34,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:50:34,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741860_1037 (size=35835) 2024-11-24T04:50:34,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/b/06c6b68efe684734ba9132558dd63768 2024-11-24T04:50:34,426 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/a/c44ba3471cdf4aa4841a4993c98678e9 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/a/c44ba3471cdf4aa4841a4993c98678e9 2024-11-24T04:50:34,435 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/a/c44ba3471cdf4aa4841a4993c98678e9, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T04:50:34,437 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/b/06c6b68efe684734ba9132558dd63768 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/b/06c6b68efe684734ba9132558dd63768 2024-11-24T04:50:34,445 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/b/06c6b68efe684734ba9132558dd63768, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T04:50:34,445 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 367677cb81527582e9c34df2dbe01a2e in 88ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T04:50:34,464 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:50:34,464 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 367677cb81527582e9c34df2dbe01a2e 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T04:50:34,473 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/b/e602a0fd414b4aa69a9260f8071aef52 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732423829519/Put/seqid=0 2024-11-24T04:50:34,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:50:34,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:50:34,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741861_1038 (size=35082) 2024-11-24T04:50:34,486 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/b/e602a0fd414b4aa69a9260f8071aef52 2024-11-24T04:50:34,516 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/c/e6e5440730f943b6b6e402a9fd1cd579 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732423829550/Put/seqid=0 2024-11-24T04:50:34,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:50:34,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:50:34,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741862_1039 (size=20825) 2024-11-24T04:50:34,529 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/c/e6e5440730f943b6b6e402a9fd1cd579 2024-11-24T04:50:34,537 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/b/e602a0fd414b4aa69a9260f8071aef52 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/b/e602a0fd414b4aa69a9260f8071aef52 2024-11-24T04:50:34,545 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/b/e602a0fd414b4aa69a9260f8071aef52, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T04:50:34,546 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/c/e6e5440730f943b6b6e402a9fd1cd579 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/c/e6e5440730f943b6b6e402a9fd1cd579 2024-11-24T04:50:34,553 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/c/e6e5440730f943b6b6e402a9fd1cd579, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T04:50:34,554 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 367677cb81527582e9c34df2dbe01a2e in 89ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T04:50:34,566 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732423829592/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:34,569 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000003002 2024-11-24T04:50:34,569 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T04:50:34,570 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 367677cb81527582e9c34df2dbe01a2e 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T04:50:34,580 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/c/7132081d640b4ccfa8881d9d161f565b is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732423829560/Put/seqid=0 2024-11-24T04:50:34,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:50:34,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:50:34,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741863_1040 (size=50301) 2024-11-24T04:50:34,594 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/c/7132081d640b4ccfa8881d9d161f565b 2024-11-24T04:50:34,602 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7132081d640b4ccfa8881d9d161f565b 2024-11-24T04:50:34,604 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/.tmp/c/7132081d640b4ccfa8881d9d161f565b as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/c/7132081d640b4ccfa8881d9d161f565b 2024-11-24T04:50:34,611 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7132081d640b4ccfa8881d9d161f565b 2024-11-24T04:50:34,611 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/c/7132081d640b4ccfa8881d9d161f565b, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T04:50:34,611 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 367677cb81527582e9c34df2dbe01a2e in 42ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T04:50:34,612 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/0000000000000003002 2024-11-24T04:50:34,613 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:34,613 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:34,614 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T04:50:34,617 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 367677cb81527582e9c34df2dbe01a2e 2024-11-24T04:50:34,620 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenIntoWAL/367677cb81527582e9c34df2dbe01a2e/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T04:50:34,621 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 367677cb81527582e9c34df2dbe01a2e; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72569741, jitterRate=0.08137340843677521}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T04:50:34,621 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 367677cb81527582e9c34df2dbe01a2e: Writing region info on filesystem at 1732423833804Initializing all the Stores at 1732423833805 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423833805Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423833806 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423833806Cleaning up temporary data from old regions at 1732423834613 (+807 ms)Region opened successfully at 1732423834621 (+8 ms) 2024-11-24T04:50:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 367677cb81527582e9c34df2dbe01a2e, disabling compactions & flushes 2024-11-24T04:50:34,698 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:34,698 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. after waiting 0 ms 2024-11-24T04:50:34,699 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:34,704 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732423828934.367677cb81527582e9c34df2dbe01a2e. 2024-11-24T04:50:34,705 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 367677cb81527582e9c34df2dbe01a2e: Waiting for close lock at 1732423834698Disabling compacts and flushes for region at 1732423834698Disabling writes for close at 1732423834698Writing region close event to WAL at 1732423834704 (+6 ms)Closed at 1732423834704 2024-11-24T04:50:34,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741857_1034 (size=93) 2024-11-24T04:50:34,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741857_1034 (size=93) 2024-11-24T04:50:34,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741857_1034 (size=93) 2024-11-24T04:50:34,715 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:34,715 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423833775) 2024-11-24T04:50:34,732 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenIntoWAL Thread=402 (was 388) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44683 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:46655 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_510594468_22 at /127.0.0.1:58060 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_510594468_22 at /127.0.0.1:42966 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44683 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46655 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_510594468_22 at /127.0.0.1:42956 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_510594468_22 at /127.0.0.1:47294 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37543 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:37543 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=428 (was 456), ProcessCount=11 (was 11), AvailableMemoryMB=11140 (was 11239) 2024-11-24T04:50:34,747 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=402, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=428, ProcessCount=11, AvailableMemoryMB=11140 2024-11-24T04:50:34,770 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:34,773 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:34,774 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:34,778 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-61682463, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-61682463, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:34,796 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-61682463/hregion-61682463.1732423834778, exclude list is [], retry=0 2024-11-24T04:50:34,800 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:34,800 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:34,802 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:34,806 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-61682463/hregion-61682463.1732423834778 2024-11-24T04:50:34,808 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:34,809 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 23375348c9d8f355ef985a3df42045d6, NAME => 'test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:34,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:50:34,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:50:34,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741865_1042 (size=43) 2024-11-24T04:50:34,828 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:34,829 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,831 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23375348c9d8f355ef985a3df42045d6 columnFamilyName a 2024-11-24T04:50:34,831 DEBUG [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:34,832 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(327): Store=23375348c9d8f355ef985a3df42045d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:34,832 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,834 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23375348c9d8f355ef985a3df42045d6 columnFamilyName b 2024-11-24T04:50:34,834 DEBUG [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:34,835 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(327): Store=23375348c9d8f355ef985a3df42045d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:34,835 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,837 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23375348c9d8f355ef985a3df42045d6 columnFamilyName c 2024-11-24T04:50:34,838 DEBUG [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:34,838 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(327): Store=23375348c9d8f355ef985a3df42045d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:34,838 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,839 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,840 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,841 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,841 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,842 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:34,843 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:34,846 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:34,846 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23375348c9d8f355ef985a3df42045d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61902574, jitterRate=-0.07757976651191711}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:34,848 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23375348c9d8f355ef985a3df42045d6: Writing region info on filesystem at 1732423834828Initializing all the Stores at 1732423834829 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423834829Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423834829Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423834829Cleaning up temporary data from old regions at 1732423834841 (+12 ms)Region opened successfully at 1732423834848 (+7 ms) 2024-11-24T04:50:34,848 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 23375348c9d8f355ef985a3df42045d6, disabling compactions & flushes 2024-11-24T04:50:34,848 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:34,848 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:34,848 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. after waiting 0 ms 2024-11-24T04:50:34,848 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:34,849 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:34,849 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 23375348c9d8f355ef985a3df42045d6: Waiting for close lock at 1732423834848Disabling compacts and flushes for region at 1732423834848Disabling writes for close at 1732423834848Writing region close event to WAL at 1732423834849 (+1 ms)Closed at 1732423834849 2024-11-24T04:50:34,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741864_1041 (size=93) 2024-11-24T04:50:34,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741864_1041 (size=93) 2024-11-24T04:50:34,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741864_1041 (size=93) 2024-11-24T04:50:34,855 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:34,856 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-61682463:(num 1732423834778) 2024-11-24T04:50:34,856 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:34,858 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:34,873 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859, exclude list is [], retry=0 2024-11-24T04:50:34,876 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:34,876 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:34,877 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:34,879 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859 2024-11-24T04:50:34,879 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:35,063 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T04:50:35,063 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:35,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:50:35,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T04:50:35,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T04:50:35,066 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:35,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T04:50:35,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:35,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741866_1043 (size=200357) 2024-11-24T04:50:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741866_1043 (size=200357) 2024-11-24T04:50:35,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741866_1043 (size=200357) 2024-11-24T04:50:35,116 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859, size=195.7 K (200357bytes) 2024-11-24T04:50:35,116 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859 2024-11-24T04:50:35,117 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859 after 1ms 2024-11-24T04:50:35,120 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:35,122 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859 took 6ms 2024-11-24T04:50:35,126 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732423834859.temp 2024-11-24T04:50:35,131 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000000001-wal.1732423834859.temp 2024-11-24T04:50:35,177 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859 so closing down 2024-11-24T04:50:35,177 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:35,177 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:35,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741867_1044 (size=200357) 2024-11-24T04:50:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741867_1044 (size=200357) 2024-11-24T04:50:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741867_1044 (size=200357) 2024-11-24T04:50:35,185 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000000001-wal.1732423834859.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-24T04:50:35,187 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000000001-wal.1732423834859.temp to hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003000 2024-11-24T04:50:35,187 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859, size=195.7 K, length=200357, corrupted=false, cancelled=false 2024-11-24T04:50:35,187 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859, journal: Splitting hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859, size=195.7 K (200357bytes) at 1732423835116Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000000001-wal.1732423834859.temp at 1732423835131 (+15 ms)Split 1024 edits, skipped 0 edits. at 1732423835143 (+12 ms)Split 2048 edits, skipped 0 edits. at 1732423835160 (+17 ms)Finishing writing output for hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859 so closing down at 1732423835177 (+17 ms)3 split writer threads finished at 1732423835177Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000000001-wal.1732423834859.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1732423835185 (+8 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000000001-wal.1732423834859.temp to hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003000 at 1732423835187 (+2 ms)Processed 3000 edits across 1 Regions in 64 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859, size=195.7 K, length=200357, corrupted=false, cancelled=false at 1732423835187 2024-11-24T04:50:35,189 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423834859 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423834859 2024-11-24T04:50:35,190 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003000 2024-11-24T04:50:35,191 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:35,193 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:35,208 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193, exclude list is [], retry=0 2024-11-24T04:50:35,212 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:35,213 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:35,213 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:35,217 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193 2024-11-24T04:50:35,220 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:35,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741868_1045 (size=200484) 2024-11-24T04:50:35,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741868_1045 (size=200484) 2024-11-24T04:50:35,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741868_1045 (size=200484) 2024-11-24T04:50:35,406 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193, size=195.8 K (200484bytes) 2024-11-24T04:50:35,406 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193 2024-11-24T04:50:35,407 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193 after 1ms 2024-11-24T04:50:35,410 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:35,412 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193 took 6ms 2024-11-24T04:50:35,422 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732423835193.temp 2024-11-24T04:50:35,425 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003001-wal.1732423835193.temp 2024-11-24T04:50:35,465 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193 so closing down 2024-11-24T04:50:35,465 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:35,468 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:35,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741869_1046 (size=200484) 2024-11-24T04:50:35,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741869_1046 (size=200484) 2024-11-24T04:50:35,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741869_1046 (size=200484) 2024-11-24T04:50:35,485 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003001-wal.1732423835193.temp (wrote 3000 edits, skipped 0 edits in 19 ms) 2024-11-24T04:50:35,486 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003001-wal.1732423835193.temp to hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000006000 2024-11-24T04:50:35,487 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193, size=195.8 K, length=200484, corrupted=false, cancelled=false 2024-11-24T04:50:35,487 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193, journal: Splitting hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193, size=195.8 K (200484bytes) at 1732423835406Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003001-wal.1732423835193.temp at 1732423835425 (+19 ms)Split 1024 edits, skipped 0 edits. at 1732423835433 (+8 ms)Split 2048 edits, skipped 0 edits. at 1732423835446 (+13 ms)Finishing writing output for hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193 so closing down at 1732423835465 (+19 ms)3 split writer threads finished at 1732423835468 (+3 ms)Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003001-wal.1732423835193.temp (wrote 3000 edits, skipped 0 edits in 19 ms) at 1732423835485 (+17 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003001-wal.1732423835193.temp to hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000006000 at 1732423835487 (+2 ms)Processed 3000 edits across 1 Regions in 69 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193, size=195.8 K, length=200484, corrupted=false, cancelled=false at 1732423835487 2024-11-24T04:50:35,491 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835193 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423835193 2024-11-24T04:50:35,492 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000006000 2024-11-24T04:50:35,493 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:35,495 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/test2727-manual,16010,1732423834769, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:35,512 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835496, exclude list is [], retry=0 2024-11-24T04:50:35,517 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:35,517 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:35,519 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:35,537 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732423834769/wal.1732423835496 2024-11-24T04:50:35,540 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:35,540 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 23375348c9d8f355ef985a3df42045d6, NAME => 'test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:35,541 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:35,541 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,541 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,543 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,546 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23375348c9d8f355ef985a3df42045d6 columnFamilyName a 2024-11-24T04:50:35,546 DEBUG [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:35,547 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(327): Store=23375348c9d8f355ef985a3df42045d6/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:35,547 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,548 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23375348c9d8f355ef985a3df42045d6 columnFamilyName b 2024-11-24T04:50:35,548 DEBUG [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:35,550 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(327): Store=23375348c9d8f355ef985a3df42045d6/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:35,550 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,552 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23375348c9d8f355ef985a3df42045d6 columnFamilyName c 2024-11-24T04:50:35,552 DEBUG [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:35,553 INFO [StoreOpener-23375348c9d8f355ef985a3df42045d6-1 {}] regionserver.HStore(327): Store=23375348c9d8f355ef985a3df42045d6/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:35,553 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,554 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,557 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003000 2024-11-24T04:50:35,560 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:35,605 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003000 2024-11-24T04:50:35,607 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000006000 2024-11-24T04:50:35,611 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:35,656 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000006000 2024-11-24T04:50:35,656 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 23375348c9d8f355ef985a3df42045d6 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T04:50:35,686 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/a/1246b867c0ed40c3974eb862806038f9 is 41, key is test2727/a:100/1732423835224/Put/seqid=0 2024-11-24T04:50:35,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:50:35,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:50:35,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741871_1048 (size=84227) 2024-11-24T04:50:35,696 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/a/1246b867c0ed40c3974eb862806038f9 2024-11-24T04:50:35,736 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/b/af287dccfba34353b58f99bb87dd541c is 41, key is test2727/b:100/1732423835272/Put/seqid=0 2024-11-24T04:50:35,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:50:35,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:50:35,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741872_1049 (size=84609) 2024-11-24T04:50:35,745 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/b/af287dccfba34353b58f99bb87dd541c 2024-11-24T04:50:35,774 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/c/be05b16fcb80411f8e4463d4eb509f44 is 41, key is test2727/c:100/1732423835331/Put/seqid=0 2024-11-24T04:50:35,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:50:35,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:50:35,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741873_1050 (size=84609) 2024-11-24T04:50:35,787 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/c/be05b16fcb80411f8e4463d4eb509f44 2024-11-24T04:50:35,796 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/a/1246b867c0ed40c3974eb862806038f9 as hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/a/1246b867c0ed40c3974eb862806038f9 2024-11-24T04:50:35,806 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/a/1246b867c0ed40c3974eb862806038f9, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T04:50:35,808 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/b/af287dccfba34353b58f99bb87dd541c as hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/b/af287dccfba34353b58f99bb87dd541c 2024-11-24T04:50:35,821 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/b/af287dccfba34353b58f99bb87dd541c, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T04:50:35,823 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/.tmp/c/be05b16fcb80411f8e4463d4eb509f44 as hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/c/be05b16fcb80411f8e4463d4eb509f44 2024-11-24T04:50:35,833 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/c/be05b16fcb80411f8e4463d4eb509f44, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T04:50:35,833 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 23375348c9d8f355ef985a3df42045d6 in 177ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T04:50:35,834 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000003000 2024-11-24T04:50:35,835 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/0000000000000006000 2024-11-24T04:50:35,837 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,837 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,838 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:35,840 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23375348c9d8f355ef985a3df42045d6 2024-11-24T04:50:35,844 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/test2727/23375348c9d8f355ef985a3df42045d6/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T04:50:35,845 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23375348c9d8f355ef985a3df42045d6; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59803671, jitterRate=-0.10885585844516754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:35,847 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23375348c9d8f355ef985a3df42045d6: Writing region info on filesystem at 1732423835541Initializing all the Stores at 1732423835543 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423835543Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423835543Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423835543Obtaining lock to block concurrent updates at 1732423835656 (+113 ms)Preparing flush snapshotting stores in 23375348c9d8f355ef985a3df42045d6 at 1732423835656Finished memstore snapshotting test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732423835656Flushing stores of test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. at 1732423835656Flushing 23375348c9d8f355ef985a3df42045d6/a: creating writer at 1732423835657 (+1 ms)Flushing 23375348c9d8f355ef985a3df42045d6/a: appending metadata at 1732423835685 (+28 ms)Flushing 23375348c9d8f355ef985a3df42045d6/a: closing flushed file at 1732423835685Flushing 23375348c9d8f355ef985a3df42045d6/b: creating writer at 1732423835704 (+19 ms)Flushing 23375348c9d8f355ef985a3df42045d6/b: appending metadata at 1732423835734 (+30 ms)Flushing 23375348c9d8f355ef985a3df42045d6/b: closing flushed file at 1732423835734Flushing 23375348c9d8f355ef985a3df42045d6/c: creating writer at 1732423835752 (+18 ms)Flushing 23375348c9d8f355ef985a3df42045d6/c: appending metadata at 1732423835774 (+22 ms)Flushing 23375348c9d8f355ef985a3df42045d6/c: closing flushed file at 1732423835774Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@531372ce: reopening flushed file at 1732423835795 (+21 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@613f543c: reopening flushed file at 1732423835806 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39224814: reopening flushed file at 1732423835822 (+16 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 23375348c9d8f355ef985a3df42045d6 in 177ms, sequenceid=6000, compaction requested=false; wal=null at 1732423835833 (+11 ms)Cleaning up temporary data from old regions at 1732423835837 (+4 ms)Region opened successfully at 1732423835846 (+9 ms) 2024-11-24T04:50:35,848 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T04:50:35,848 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 23375348c9d8f355ef985a3df42045d6, disabling compactions & flushes 2024-11-24T04:50:35,848 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:35,849 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:35,849 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. after waiting 0 ms 2024-11-24T04:50:35,849 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:35,851 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732423834771.23375348c9d8f355ef985a3df42045d6. 2024-11-24T04:50:35,851 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 23375348c9d8f355ef985a3df42045d6: Waiting for close lock at 1732423835848Disabling compacts and flushes for region at 1732423835848Disabling writes for close at 1732423835849 (+1 ms)Writing region close event to WAL at 1732423835850 (+1 ms)Closed at 1732423835851 (+1 ms) 2024-11-24T04:50:35,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741870_1047 (size=93) 2024-11-24T04:50:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741870_1047 (size=93) 2024-11-24T04:50:35,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741870_1047 (size=93) 2024-11-24T04:50:35,858 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:35,858 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423835496) 2024-11-24T04:50:35,874 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#test2727 Thread=406 (was 402) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:47464 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:58060 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43046 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=428 (was 428), ProcessCount=11 (was 11), AvailableMemoryMB=10938 (was 11140) 2024-11-24T04:50:35,886 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=406, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=428, ProcessCount=11, AvailableMemoryMB=10937 2024-11-24T04:50:35,901 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:35,908 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:35,909 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732423835908 2024-11-24T04:50:35,921 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908 2024-11-24T04:50:35,936 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:35,945 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 756c6abb95ecb9b56ba7e84c7e3d23f4, NAME => 'testSequentialEditLogSeqNum,,1732423835901.756c6abb95ecb9b56ba7e84c7e3d23f4.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:35,945 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732423835901.756c6abb95ecb9b56ba7e84c7e3d23f4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:35,945 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,945 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,947 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4 doesn't exist for region: 756c6abb95ecb9b56ba7e84c7e3d23f4 on table testSequentialEditLogSeqNum 2024-11-24T04:50:35,948 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 756c6abb95ecb9b56ba7e84c7e3d23f4 on table testSequentialEditLogSeqNum 2024-11-24T04:50:35,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:50:35,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:50:35,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741875_1052 (size=62) 2024-11-24T04:50:35,974 INFO [StoreOpener-756c6abb95ecb9b56ba7e84c7e3d23f4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,976 INFO [StoreOpener-756c6abb95ecb9b56ba7e84c7e3d23f4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756c6abb95ecb9b56ba7e84c7e3d23f4 columnFamilyName a 2024-11-24T04:50:35,976 DEBUG [StoreOpener-756c6abb95ecb9b56ba7e84c7e3d23f4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:35,977 INFO [StoreOpener-756c6abb95ecb9b56ba7e84c7e3d23f4-1 {}] regionserver.HStore(327): Store=756c6abb95ecb9b56ba7e84c7e3d23f4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:35,977 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,979 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,979 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,982 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 756c6abb95ecb9b56ba7e84c7e3d23f4 2024-11-24T04:50:35,984 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:35,985 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 756c6abb95ecb9b56ba7e84c7e3d23f4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73283226, jitterRate=0.0920051634311676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:35,986 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 756c6abb95ecb9b56ba7e84c7e3d23f4: Writing region info on filesystem at 1732423835945Initializing all the Stores at 1732423835973 (+28 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423835973Cleaning up temporary data from old regions at 1732423835979 (+6 ms)Region opened successfully at 1732423835986 (+7 ms) 2024-11-24T04:50:36,003 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 756c6abb95ecb9b56ba7e84c7e3d23f4 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T04:50:36,030 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/.tmp/a/5a2ad1da7bd64cee860a9f1b0a930f66 is 81, key is testSequentialEditLogSeqNum/a:x0/1732423835986/Put/seqid=0 2024-11-24T04:50:36,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:50:36,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:50:36,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741876_1053 (size=5833) 2024-11-24T04:50:36,039 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/.tmp/a/5a2ad1da7bd64cee860a9f1b0a930f66 2024-11-24T04:50:36,049 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/.tmp/a/5a2ad1da7bd64cee860a9f1b0a930f66 as hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/a/5a2ad1da7bd64cee860a9f1b0a930f66 2024-11-24T04:50:36,058 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/a/5a2ad1da7bd64cee860a9f1b0a930f66, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T04:50:36,060 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 756c6abb95ecb9b56ba7e84c7e3d23f4 in 58ms, sequenceid=13, compaction requested=false 2024-11-24T04:50:36,060 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 756c6abb95ecb9b56ba7e84c7e3d23f4: 2024-11-24T04:50:36,068 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:50:36,068 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:50:36,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:50:36,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:50:36,069 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T04:50:36,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741874_1051 (size=1616) 2024-11-24T04:50:36,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741874_1051 (size=1616) 2024-11-24T04:50:36,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741874_1051 (size=1616) 2024-11-24T04:50:36,094 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908, size=1.6 K (1616bytes) 2024-11-24T04:50:36,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908 2024-11-24T04:50:36,095 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908 after 0ms 2024-11-24T04:50:36,098 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:36,098 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908 took 4ms 2024-11-24T04:50:36,101 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908 so closing down 2024-11-24T04:50:36,101 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:36,104 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423835908.temp 2024-11-24T04:50:36,106 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000003-wal.1732423835908.temp 2024-11-24T04:50:36,106 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:36,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741877_1054 (size=1175) 2024-11-24T04:50:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741877_1054 (size=1175) 2024-11-24T04:50:36,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741877_1054 (size=1175) 2024-11-24T04:50:36,116 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000003-wal.1732423835908.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:36,117 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000003-wal.1732423835908.temp to hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000020 2024-11-24T04:50:36,117 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908, size=1.6 K, length=1616, corrupted=false, cancelled=false 2024-11-24T04:50:36,118 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908, journal: Splitting hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908, size=1.6 K (1616bytes) at 1732423836094Finishing writing output for hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908 so closing down at 1732423836101 (+7 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000003-wal.1732423835908.temp at 1732423836106 (+5 ms)3 split writer threads finished at 1732423836106Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000003-wal.1732423835908.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732423836116 (+10 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000003-wal.1732423835908.temp to hdfs://localhost:44683/hbase/data/default/testSequentialEditLogSeqNum/756c6abb95ecb9b56ba7e84c7e3d23f4/recovered.edits/0000000000000000020 at 1732423836117 (+1 ms)Processed 17 edits across 1 Regions in 19 ms; skipped=2; WAL=hdfs://localhost:44683/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732423835900/wal.1732423835908, size=1.6 K, length=1616, corrupted=false, cancelled=false at 1732423836118 (+1 ms) 2024-11-24T04:50:36,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testSequentialEditLogSeqNum Thread=411 (was 406) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:58060 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43046 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=428 (was 428), ProcessCount=11 (was 11), AvailableMemoryMB=10910 (was 10937) 2024-11-24T04:50:36,150 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=411, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=428, ProcessCount=11, AvailableMemoryMB=10904 2024-11-24T04:50:36,166 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:36,169 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:36,197 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:36,200 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-27993547, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-27993547, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:36,215 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-27993547/hregion-27993547.1732423836201, exclude list is [], retry=0 2024-11-24T04:50:36,219 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:36,219 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:36,219 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:36,223 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-27993547/hregion-27993547.1732423836201 2024-11-24T04:50:36,223 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:36,224 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2d90e051281b752408fb8d2f830bd242, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:36,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:50:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:50:36,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741879_1056 (size=70) 2024-11-24T04:50:36,285 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:36,288 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,295 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName a 2024-11-24T04:50:36,296 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:36,297 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:36,302 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,304 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName b 2024-11-24T04:50:36,304 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:36,305 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:36,306 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,309 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName c 2024-11-24T04:50:36,309 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:36,310 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:36,310 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,311 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,312 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,313 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,313 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,314 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:36,316 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,320 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:36,321 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d90e051281b752408fb8d2f830bd242; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63450189, jitterRate=-0.054518505930900574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:36,322 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d90e051281b752408fb8d2f830bd242: Writing region info on filesystem at 1732423836285Initializing all the Stores at 1732423836288 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423836288Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423836288Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423836288Cleaning up temporary data from old regions at 1732423836313 (+25 ms)Region opened successfully at 1732423836322 (+9 ms) 2024-11-24T04:50:36,322 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d90e051281b752408fb8d2f830bd242, disabling compactions & flushes 2024-11-24T04:50:36,323 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:36,323 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:36,323 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. after waiting 0 ms 2024-11-24T04:50:36,323 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:36,324 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:36,324 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d90e051281b752408fb8d2f830bd242: Waiting for close lock at 1732423836322Disabling compacts and flushes for region at 1732423836322Disabling writes for close at 1732423836323 (+1 ms)Writing region close event to WAL at 1732423836324 (+1 ms)Closed at 1732423836324 2024-11-24T04:50:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741878_1055 (size=93) 2024-11-24T04:50:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741878_1055 (size=93) 2024-11-24T04:50:36,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741878_1055 (size=93) 2024-11-24T04:50:36,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:36,336 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-27993547:(num 1732423836201) 2024-11-24T04:50:36,337 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:36,339 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:36,354 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340, exclude list is [], retry=0 2024-11-24T04:50:36,357 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:36,358 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:36,358 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:36,360 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 2024-11-24T04:50:36,360 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:36,360 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d90e051281b752408fb8d2f830bd242, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:36,360 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:36,361 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,361 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,364 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,366 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName a 2024-11-24T04:50:36,366 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:36,367 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:36,367 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,368 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName b 2024-11-24T04:50:36,369 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:36,369 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:36,369 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,370 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName c 2024-11-24T04:50:36,370 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:36,371 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:36,371 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,372 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,373 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,374 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,374 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,375 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:36,376 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:36,377 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d90e051281b752408fb8d2f830bd242; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68769698, jitterRate=0.02474835515022278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:36,379 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d90e051281b752408fb8d2f830bd242: Writing region info on filesystem at 1732423836361Initializing all the Stores at 1732423836362 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423836362Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423836364 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423836364Cleaning up temporary data from old regions at 1732423836374 (+10 ms)Region opened successfully at 1732423836378 (+4 ms) 2024-11-24T04:50:36,384 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732423836383/Put/seqid=0 2024-11-24T04:50:36,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:50:36,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:50:36,396 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44683/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 2d90e051281b752408fb8d2f830bd242/a 2024-11-24T04:50:36,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741881_1058 (size=4826) 2024-11-24T04:50:36,405 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T04:50:36,405 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:50:36,405 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2d90e051281b752408fb8d2f830bd242: 2024-11-24T04:50:36,407 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/a/92f54b33129140f0b1f7e3b8d623eb59_SeqId_3_ 2024-11-24T04:50:36,409 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44683/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2d90e051281b752408fb8d2f830bd242/a as hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/a/92f54b33129140f0b1f7e3b8d623eb59_SeqId_3_ - updating store file list. 2024-11-24T04:50:36,416 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 92f54b33129140f0b1f7e3b8d623eb59_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:36,417 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/a/92f54b33129140f0b1f7e3b8d623eb59_SeqId_3_ into 2d90e051281b752408fb8d2f830bd242/a 2024-11-24T04:50:36,417 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44683/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 2d90e051281b752408fb8d2f830bd242/a (new location: hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/a/92f54b33129140f0b1f7e3b8d623eb59_SeqId_3_) 2024-11-24T04:50:36,461 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340, size=0 (0bytes) 2024-11-24T04:50:36,461 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 might be still open, length is 0 2024-11-24T04:50:36,461 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 2024-11-24T04:50:36,462 WARN [IPC Server handler 4 on default port 44683 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T04:50:36,462 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 after 1ms 2024-11-24T04:50:38,874 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:47556 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47556 dst: /127.0.0.1:43517 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43517 remote=/127.0.0.1:47556]. Total timeout mills is 60000, 57550 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:38,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43150 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46055:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43150 dst: /127.0.0.1:46055 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:38,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:58264 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58264 dst: /127.0.0.1:42305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:38,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741880_1059 (size=434) 2024-11-24T04:50:38,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741880_1059 (size=434) 2024-11-24T04:50:40,463 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 after 4002ms 2024-11-24T04:50:40,466 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:40,467 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 took 4006ms 2024-11-24T04:50:40,469 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340; continuing. 2024-11-24T04:50:40,469 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 so closing down 2024-11-24T04:50:40,469 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:40,471 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732423836340.temp 2024-11-24T04:50:40,473 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005-wal.1732423836340.temp 2024-11-24T04:50:40,473 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:40,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741882_1060 (size=236) 2024-11-24T04:50:40,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741882_1060 (size=236) 2024-11-24T04:50:40,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741882_1060 (size=236) 2024-11-24T04:50:40,484 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005-wal.1732423836340.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:40,485 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005-wal.1732423836340.temp to hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005 2024-11-24T04:50:40,486 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 19 ms; skipped=1; WAL=hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:50:40,486 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340, journal: Splitting hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340, size=0 (0bytes) at 1732423836461Finishing writing output for hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 so closing down at 1732423840469 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005-wal.1732423836340.temp at 1732423840473 (+4 ms)3 split writer threads finished at 1732423840473Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005-wal.1732423836340.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732423840484 (+11 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005-wal.1732423836340.temp to hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005 at 1732423840485 (+1 ms)Processed 2 edits across 1 Regions in 19 ms; skipped=1; WAL=hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340, size=0, length=0, corrupted=false, cancelled=false at 1732423840486 (+1 ms) 2024-11-24T04:50:40,488 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423836340 2024-11-24T04:50:40,489 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005 2024-11-24T04:50:40,489 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:40,490 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:40,504 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423840491, exclude list is [], retry=0 2024-11-24T04:50:40,507 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:40,508 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:40,508 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:40,513 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423840491 2024-11-24T04:50:40,513 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:40,513 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d90e051281b752408fb8d2f830bd242, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:40,513 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:40,514 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,514 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,515 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,517 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName a 2024-11-24T04:50:40,517 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:40,524 DEBUG [StoreFileOpener-2d90e051281b752408fb8d2f830bd242-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 92f54b33129140f0b1f7e3b8d623eb59_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:40,524 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/a/92f54b33129140f0b1f7e3b8d623eb59_SeqId_3_ 2024-11-24T04:50:40,524 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:40,524 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,525 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName b 2024-11-24T04:50:40,526 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:40,526 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:40,526 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,527 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d90e051281b752408fb8d2f830bd242 columnFamilyName c 2024-11-24T04:50:40,528 DEBUG [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:40,528 INFO [StoreOpener-2d90e051281b752408fb8d2f830bd242-1 {}] regionserver.HStore(327): Store=2d90e051281b752408fb8d2f830bd242/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:40,528 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,529 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,531 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,532 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005 2024-11-24T04:50:40,535 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:40,536 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005 2024-11-24T04:50:40,536 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2d90e051281b752408fb8d2f830bd242 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T04:50:40,553 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/.tmp/a/afc3196daa914ca495100de03cddc90a is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732423836423/Put/seqid=0 2024-11-24T04:50:40,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:50:40,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:50:40,561 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/.tmp/a/afc3196daa914ca495100de03cddc90a 2024-11-24T04:50:40,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741884_1062 (size=5149) 2024-11-24T04:50:40,568 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/.tmp/a/afc3196daa914ca495100de03cddc90a as hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/a/afc3196daa914ca495100de03cddc90a 2024-11-24T04:50:40,574 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/a/afc3196daa914ca495100de03cddc90a, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T04:50:40,574 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2d90e051281b752408fb8d2f830bd242 in 38ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T04:50:40,575 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/0000000000000000005 2024-11-24T04:50:40,576 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,576 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,577 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:40,579 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2d90e051281b752408fb8d2f830bd242 2024-11-24T04:50:40,581 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/2d90e051281b752408fb8d2f830bd242/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T04:50:40,582 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2d90e051281b752408fb8d2f830bd242; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65618861, jitterRate=-0.02220277488231659}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:40,582 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2d90e051281b752408fb8d2f830bd242: Writing region info on filesystem at 1732423840514Initializing all the Stores at 1732423840515 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423840515Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423840515Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423840515Obtaining lock to block concurrent updates at 1732423840536 (+21 ms)Preparing flush snapshotting stores in 2d90e051281b752408fb8d2f830bd242 at 1732423840536Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732423840536Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. at 1732423840536Flushing 2d90e051281b752408fb8d2f830bd242/a: creating writer at 1732423840536Flushing 2d90e051281b752408fb8d2f830bd242/a: appending metadata at 1732423840552 (+16 ms)Flushing 2d90e051281b752408fb8d2f830bd242/a: closing flushed file at 1732423840553 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21e029b0: reopening flushed file at 1732423840566 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 2d90e051281b752408fb8d2f830bd242 in 38ms, sequenceid=5, compaction requested=false; wal=null at 1732423840574 (+8 ms)Cleaning up temporary data from old regions at 1732423840576 (+2 ms)Region opened successfully at 1732423840582 (+6 ms) 2024-11-24T04:50:40,587 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2d90e051281b752408fb8d2f830bd242, disabling compactions & flushes 2024-11-24T04:50:40,587 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:40,587 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:40,587 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. after waiting 0 ms 2024-11-24T04:50:40,587 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:40,588 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732423836167.2d90e051281b752408fb8d2f830bd242. 2024-11-24T04:50:40,588 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2d90e051281b752408fb8d2f830bd242: Waiting for close lock at 1732423840587Disabling compacts and flushes for region at 1732423840587Disabling writes for close at 1732423840587Writing region close event to WAL at 1732423840588 (+1 ms)Closed at 1732423840588 2024-11-24T04:50:40,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741883_1061 (size=93) 2024-11-24T04:50:40,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741883_1061 (size=93) 2024-11-24T04:50:40,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741883_1061 (size=93) 2024-11-24T04:50:40,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:40,594 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423840491) 2024-11-24T04:50:40,606 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testRegionMadeOfBulkLoadedFilesOnly Thread=414 (was 411) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2126463691_22 at /127.0.0.1:43182 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44683 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2126463691_22 at /127.0.0.1:58292 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2126463691_22 at /127.0.0.1:47580 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44683 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=410 (was 428), ProcessCount=11 (was 11), AvailableMemoryMB=10837 (was 10904) 2024-11-24T04:50:40,618 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=414, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=410, ProcessCount=11, AvailableMemoryMB=10836 2024-11-24T04:50:40,632 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:40,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T04:50:40,640 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4464c5b832df,36681,1732423824726 2024-11-24T04:50:40,642 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d6d2faa 2024-11-24T04:50:40,643 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T04:50:40,645 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T04:50:40,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T04:50:40,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T04:50:40,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T04:50:40,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T04:50:40,660 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:40,662 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T04:50:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:50:40,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:50:40,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:50:40,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741885_1063 (size=694) 2024-11-24T04:50:40,682 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8a87c32a42ac0dc0089a1412f6f2216d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322 2024-11-24T04:50:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:50:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:50:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741886_1064 (size=77) 2024-11-24T04:50:40,696 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:40,696 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 8a87c32a42ac0dc0089a1412f6f2216d, disabling compactions & flushes 2024-11-24T04:50:40,696 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:40,697 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:40,697 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. after waiting 0 ms 2024-11-24T04:50:40,697 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:40,697 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:40,697 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8a87c32a42ac0dc0089a1412f6f2216d: Waiting for close lock at 1732423840696Disabling compacts and flushes for region at 1732423840696Disabling writes for close at 1732423840697 (+1 ms)Writing region close event to WAL at 1732423840697Closed at 1732423840697 2024-11-24T04:50:40,698 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T04:50:40,704 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732423840698"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732423840698"}]},"ts":"1732423840698"} 2024-11-24T04:50:40,707 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T04:50:40,709 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T04:50:40,711 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732423840709"}]},"ts":"1732423840709"} 2024-11-24T04:50:40,714 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T04:50:40,715 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4464c5b832df=0} racks are {/default-rack=0} 2024-11-24T04:50:40,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:50:40,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:50:40,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T04:50:40,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:50:40,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:50:40,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T04:50:40,717 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:50:40,717 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:50:40,717 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T04:50:40,717 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:50:40,718 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN}] 2024-11-24T04:50:40,720 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN 2024-11-24T04:50:40,721 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN; state=OFFLINE, location=4464c5b832df,36869,1732423825485; forceNewPlan=false, retain=false 2024-11-24T04:50:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:50:40,873 INFO [4464c5b832df:36681 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:50:40,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPENING, regionLocation=4464c5b832df,36869,1732423825485 2024-11-24T04:50:40,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN because future has completed 2024-11-24T04:50:40,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485}] 2024-11-24T04:50:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:50:41,031 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:50:41,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38179, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:50:41,038 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,039 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a87c32a42ac0dc0089a1412f6f2216d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:41,039 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,039 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:41,039 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,039 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,041 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,043 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf1 2024-11-24T04:50:41,044 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:41,044 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:41,044 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,046 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf2 2024-11-24T04:50:41,046 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:41,047 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:41,047 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,048 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,048 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,049 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,049 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,050 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:50:41,052 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,055 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:41,056 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8a87c32a42ac0dc0089a1412f6f2216d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65600590, jitterRate=-0.02247503399848938}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:50:41,056 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,057 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8a87c32a42ac0dc0089a1412f6f2216d: Running coprocessor pre-open hook at 1732423841040Writing region info on filesystem at 1732423841040Initializing all the Stores at 1732423841041 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423841041Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423841041Cleaning up temporary data from old regions at 1732423841049 (+8 ms)Running coprocessor post-open hooks at 1732423841056 (+7 ms)Region opened successfully at 1732423841057 (+1 ms) 2024-11-24T04:50:41,059 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., pid=6, masterSystemTime=1732423841031 2024-11-24T04:50:41,063 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,063 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,064 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPEN, openSeqNum=2, regionLocation=4464c5b832df,36869,1732423825485 2024-11-24T04:50:41,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485 because future has completed 2024-11-24T04:50:41,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T04:50:41,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485 in 192 msec 2024-11-24T04:50:41,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T04:50:41,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN in 357 msec 2024-11-24T04:50:41,083 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T04:50:41,083 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732423841083"}]},"ts":"1732423841083"} 2024-11-24T04:50:41,087 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T04:50:41,090 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T04:50:41,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 440 msec 2024-11-24T04:50:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T04:50:41,299 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T04:50:41,299 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T04:50:41,300 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:50:41,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T04:50:41,306 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T04:50:41,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T04:50:41,318 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=2] 2024-11-24T04:50:41,320 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:50:41,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37728, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:50:41,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8a87c32a42ac0dc0089a1412f6f2216d, source=4464c5b832df,36869,1732423825485, destination=4464c5b832df,46377,1732423825594, warming up region on 4464c5b832df,46377,1732423825594 2024-11-24T04:50:41,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:50:41,344 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8a87c32a42ac0dc0089a1412f6f2216d, source=4464c5b832df,36869,1732423825485, destination=4464c5b832df,46377,1732423825594, running balancer 2024-11-24T04:50:41,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE 2024-11-24T04:50:41,345 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE 2024-11-24T04:50:41,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49045, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:50:41,348 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=CLOSING, regionLocation=4464c5b832df,36869,1732423825485 2024-11-24T04:50:41,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE because future has completed 2024-11-24T04:50:41,351 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:50:41,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485}] 2024-11-24T04:50:41,352 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8a87c32a42ac0dc0089a1412f6f2216d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:41,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:41,356 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,359 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf1 2024-11-24T04:50:41,359 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:41,360 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:41,360 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,361 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf2 2024-11-24T04:50:41,361 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:41,362 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(1722): Closing 8a87c32a42ac0dc0089a1412f6f2216d, disabling compactions & flushes 2024-11-24T04:50:41,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. after waiting 0 ms 2024-11-24T04:50:41,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,364 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] regionserver.HRegion(1676): Region close journal for 8a87c32a42ac0dc0089a1412f6f2216d: Waiting for close lock at 1732423841362Disabling compacts and flushes for region at 1732423841362Disabling writes for close at 1732423841362Writing region close event to WAL at 1732423841364 (+2 ms)Closed at 1732423841364 2024-11-24T04:50:41,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T04:50:41,511 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,511 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:50:41,512 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8a87c32a42ac0dc0089a1412f6f2216d, disabling compactions & flushes 2024-11-24T04:50:41,512 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,512 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,512 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. after waiting 0 ms 2024-11-24T04:50:41,512 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,512 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 8a87c32a42ac0dc0089a1412f6f2216d 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T04:50:41,530 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf1/693201f334d24733a0cb439fb46c8ad8 is 35, key is r1/cf1:q/1732423841323/Put/seqid=0 2024-11-24T04:50:41,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:50:41,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:50:41,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741887_1065 (size=4783) 2024-11-24T04:50:41,543 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf1/693201f334d24733a0cb439fb46c8ad8 2024-11-24T04:50:41,552 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf1/693201f334d24733a0cb439fb46c8ad8 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8 2024-11-24T04:50:41,560 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T04:50:41,562 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 8a87c32a42ac0dc0089a1412f6f2216d in 50ms, sequenceid=5, compaction requested=false 2024-11-24T04:50:41,562 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T04:50:41,569 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T04:50:41,571 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,572 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8a87c32a42ac0dc0089a1412f6f2216d: Waiting for close lock at 1732423841512Running coprocessor pre-close hooks at 1732423841512Disabling compacts and flushes for region at 1732423841512Disabling writes for close at 1732423841512Obtaining lock to block concurrent updates at 1732423841512Preparing flush snapshotting stores in 8a87c32a42ac0dc0089a1412f6f2216d at 1732423841512Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732423841513 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. at 1732423841514 (+1 ms)Flushing 8a87c32a42ac0dc0089a1412f6f2216d/cf1: creating writer at 1732423841514Flushing 8a87c32a42ac0dc0089a1412f6f2216d/cf1: appending metadata at 1732423841530 (+16 ms)Flushing 8a87c32a42ac0dc0089a1412f6f2216d/cf1: closing flushed file at 1732423841530Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6500714c: reopening flushed file at 1732423841550 (+20 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 8a87c32a42ac0dc0089a1412f6f2216d in 50ms, sequenceid=5, compaction requested=false at 1732423841562 (+12 ms)Writing region close event to WAL at 1732423841564 (+2 ms)Running coprocessor post-close hooks at 1732423841569 (+5 ms)Closed at 1732423841571 (+2 ms) 2024-11-24T04:50:41,572 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8a87c32a42ac0dc0089a1412f6f2216d move to 4464c5b832df,46377,1732423825594 record at close sequenceid=5 2024-11-24T04:50:41,575 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=CLOSED 2024-11-24T04:50:41,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485 because future has completed 2024-11-24T04:50:41,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T04:50:41,593 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485 in 234 msec 2024-11-24T04:50:41,595 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE; state=CLOSED, location=4464c5b832df,46377,1732423825594; forceNewPlan=false, retain=false 2024-11-24T04:50:41,745 INFO [4464c5b832df:36681 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:50:41,746 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPENING, regionLocation=4464c5b832df,46377,1732423825594 2024-11-24T04:50:41,748 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE because future has completed 2024-11-24T04:50:41,749 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,46377,1732423825594}] 2024-11-24T04:50:41,906 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,907 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a87c32a42ac0dc0089a1412f6f2216d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:41,907 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,907 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:41,907 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,907 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,909 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,910 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf1 2024-11-24T04:50:41,910 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:41,917 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8 2024-11-24T04:50:41,917 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:41,917 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,919 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf2 2024-11-24T04:50:41,919 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:41,920 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:41,920 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,921 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,922 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,923 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,923 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,924 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:50:41,925 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,926 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8a87c32a42ac0dc0089a1412f6f2216d; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66991126, jitterRate=-0.0017544329166412354}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:50:41,927 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:41,927 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8a87c32a42ac0dc0089a1412f6f2216d: Running coprocessor pre-open hook at 1732423841907Writing region info on filesystem at 1732423841907Initializing all the Stores at 1732423841908 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423841908Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423841909 (+1 ms)Cleaning up temporary data from old regions at 1732423841923 (+14 ms)Running coprocessor post-open hooks at 1732423841927 (+4 ms)Region opened successfully at 1732423841927 2024-11-24T04:50:41,928 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., pid=9, masterSystemTime=1732423841902 2024-11-24T04:50:41,931 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,931 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:41,933 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPEN, openSeqNum=9, regionLocation=4464c5b832df,46377,1732423825594 2024-11-24T04:50:41,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,46377,1732423825594 because future has completed 2024-11-24T04:50:41,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T04:50:41,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,46377,1732423825594 in 188 msec 2024-11-24T04:50:41,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE in 596 msec 2024-11-24T04:50:41,951 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:50:41,953 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40286, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:50:41,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:37728 deadline: 1732423901957, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=46377 startCode=1732423825594. As of locationSeqNum=5. 2024-11-24T04:50:41,987 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=46377 startCode=1732423825594. As of locationSeqNum=5. 2024-11-24T04:50:41,987 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=46377 startCode=1732423825594. As of locationSeqNum=5. 2024-11-24T04:50:41,988 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,46377,1732423825594, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=46377 startCode=1732423825594. As of locationSeqNum=5. 2024-11-24T04:50:42,097 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T04:50:42,100 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T04:50:42,109 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8a87c32a42ac0dc0089a1412f6f2216d 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T04:50:42,131 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf1/4f96aac3d0cb42e8904d6942248c8265 is 29, key is r1/cf1:/1732423842101/DeleteFamily/seqid=0 2024-11-24T04:50:42,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:50:42,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:50:42,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741888_1066 (size=4906) 2024-11-24T04:50:42,141 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf1/4f96aac3d0cb42e8904d6942248c8265 2024-11-24T04:50:42,149 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f96aac3d0cb42e8904d6942248c8265 2024-11-24T04:50:42,164 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf2/b2d305c9b0da40dcbc16e7cd983b21c7 is 29, key is r1/cf2:/1732423842101/DeleteFamily/seqid=0 2024-11-24T04:50:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:50:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:50:42,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741889_1067 (size=4906) 2024-11-24T04:50:42,173 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf2/b2d305c9b0da40dcbc16e7cd983b21c7 2024-11-24T04:50:42,181 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2d305c9b0da40dcbc16e7cd983b21c7 2024-11-24T04:50:42,182 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf1/4f96aac3d0cb42e8904d6942248c8265 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/4f96aac3d0cb42e8904d6942248c8265 2024-11-24T04:50:42,190 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f96aac3d0cb42e8904d6942248c8265 2024-11-24T04:50:42,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/4f96aac3d0cb42e8904d6942248c8265, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T04:50:42,191 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf2/b2d305c9b0da40dcbc16e7cd983b21c7 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/b2d305c9b0da40dcbc16e7cd983b21c7 2024-11-24T04:50:42,199 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2d305c9b0da40dcbc16e7cd983b21c7 2024-11-24T04:50:42,199 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/b2d305c9b0da40dcbc16e7cd983b21c7, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T04:50:42,200 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 8a87c32a42ac0dc0089a1412f6f2216d in 91ms, sequenceid=12, compaction requested=false 2024-11-24T04:50:42,200 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8a87c32a42ac0dc0089a1412f6f2216d: 2024-11-24T04:50:42,203 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T04:50:42,205 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8a87c32a42ac0dc0089a1412f6f2216d/cf1 is initiating major compaction (all files) 2024-11-24T04:50:42,206 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:50:42,206 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:42,206 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8a87c32a42ac0dc0089a1412f6f2216d/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,207 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8, hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/4f96aac3d0cb42e8904d6942248c8265] into tmpdir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp, totalSize=9.5 K 2024-11-24T04:50:42,208 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 693201f334d24733a0cb439fb46c8ad8, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732423841323 2024-11-24T04:50:42,209 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4f96aac3d0cb42e8904d6942248c8265, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T04:50:42,222 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 8a87c32a42ac0dc0089a1412f6f2216d#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T04:50:42,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:50:42,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:50:42,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741890_1068 (size=4626) 2024-11-24T04:50:42,241 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf1/bd61afa88bea4267b07e8297b58cbb13 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/bd61afa88bea4267b07e8297b58cbb13 2024-11-24T04:50:42,258 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 8a87c32a42ac0dc0089a1412f6f2216d/cf1 of 8a87c32a42ac0dc0089a1412f6f2216d into bd61afa88bea4267b07e8297b58cbb13(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:50:42,258 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8a87c32a42ac0dc0089a1412f6f2216d: 2024-11-24T04:50:42,258 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T04:50:42,258 DEBUG [Time-limited test {}] regionserver.HStore(1541): 8a87c32a42ac0dc0089a1412f6f2216d/cf2 is initiating major compaction (all files) 2024-11-24T04:50:42,259 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T04:50:42,259 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T04:50:42,259 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 8a87c32a42ac0dc0089a1412f6f2216d/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,259 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/b2d305c9b0da40dcbc16e7cd983b21c7] into tmpdir=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp, totalSize=4.8 K 2024-11-24T04:50:42,260 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting b2d305c9b0da40dcbc16e7cd983b21c7, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T04:50:42,266 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 8a87c32a42ac0dc0089a1412f6f2216d#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T04:50:42,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:50:42,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:50:42,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741891_1069 (size=4592) 2024-11-24T04:50:42,282 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/.tmp/cf2/ac90313313d24a9792cb461bb5cec2c5 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/ac90313313d24a9792cb461bb5cec2c5 2024-11-24T04:50:42,291 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 8a87c32a42ac0dc0089a1412f6f2216d/cf2 of 8a87c32a42ac0dc0089a1412f6f2216d into ac90313313d24a9792cb461bb5cec2c5(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:50:42,291 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 8a87c32a42ac0dc0089a1412f6f2216d: 2024-11-24T04:50:42,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8a87c32a42ac0dc0089a1412f6f2216d, source=4464c5b832df,46377,1732423825594, destination=4464c5b832df,36869,1732423825485, warming up region on 4464c5b832df,36869,1732423825485 2024-11-24T04:50:42,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8a87c32a42ac0dc0089a1412f6f2216d, source=4464c5b832df,46377,1732423825594, destination=4464c5b832df,36869,1732423825485, running balancer 2024-11-24T04:50:42,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE 2024-11-24T04:50:42,298 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE 2024-11-24T04:50:42,300 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=CLOSING, regionLocation=4464c5b832df,46377,1732423825594 2024-11-24T04:50:42,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8a87c32a42ac0dc0089a1412f6f2216d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:42,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:42,301 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,302 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf1 2024-11-24T04:50:42,303 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:42,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE because future has completed 2024-11-24T04:50:42,303 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T04:50:42,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,46377,1732423825594}] 2024-11-24T04:50:42,312 INFO [StoreFileOpener-8a87c32a42ac0dc0089a1412f6f2216d-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 4f96aac3d0cb42e8904d6942248c8265 2024-11-24T04:50:42,313 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/4f96aac3d0cb42e8904d6942248c8265 2024-11-24T04:50:42,320 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8 2024-11-24T04:50:42,327 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/bd61afa88bea4267b07e8297b58cbb13 2024-11-24T04:50:42,327 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:42,328 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,329 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf2 2024-11-24T04:50:42,329 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:42,337 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/ac90313313d24a9792cb461bb5cec2c5 2024-11-24T04:50:42,343 INFO [StoreFileOpener-8a87c32a42ac0dc0089a1412f6f2216d-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b2d305c9b0da40dcbc16e7cd983b21c7 2024-11-24T04:50:42,343 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/b2d305c9b0da40dcbc16e7cd983b21c7 2024-11-24T04:50:42,343 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(1722): Closing 8a87c32a42ac0dc0089a1412f6f2216d, disabling compactions & flushes 2024-11-24T04:50:42,343 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. after waiting 0 ms 2024-11-24T04:50:42,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36869 {}] regionserver.HRegion(1676): Region close journal for 8a87c32a42ac0dc0089a1412f6f2216d: Waiting for close lock at 1732423842343Disabling compacts and flushes for region at 1732423842343Disabling writes for close at 1732423842343Writing region close event to WAL at 1732423842345 (+2 ms)Closed at 1732423842345 2024-11-24T04:50:42,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T04:50:42,458 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,458 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T04:50:42,458 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 8a87c32a42ac0dc0089a1412f6f2216d, disabling compactions & flushes 2024-11-24T04:50:42,458 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,458 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,458 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. after waiting 0 ms 2024-11-24T04:50:42,458 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,459 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8, hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/4f96aac3d0cb42e8904d6942248c8265] to archive 2024-11-24T04:50:42,462 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:50:42,467 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8 to hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/693201f334d24733a0cb439fb46c8ad8 2024-11-24T04:50:42,469 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/4f96aac3d0cb42e8904d6942248c8265 to hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/4f96aac3d0cb42e8904d6942248c8265 2024-11-24T04:50:42,484 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/b2d305c9b0da40dcbc16e7cd983b21c7] to archive 2024-11-24T04:50:42,486 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:50:42,488 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/b2d305c9b0da40dcbc16e7cd983b21c7 to hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/b2d305c9b0da40dcbc16e7cd983b21c7 2024-11-24T04:50:42,497 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T04:50:42,498 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,498 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 8a87c32a42ac0dc0089a1412f6f2216d: Waiting for close lock at 1732423842458Running coprocessor pre-close hooks at 1732423842458Disabling compacts and flushes for region at 1732423842458Disabling writes for close at 1732423842458Writing region close event to WAL at 1732423842492 (+34 ms)Running coprocessor post-close hooks at 1732423842498 (+6 ms)Closed at 1732423842498 2024-11-24T04:50:42,498 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 8a87c32a42ac0dc0089a1412f6f2216d move to 4464c5b832df,36869,1732423825485 record at close sequenceid=12 2024-11-24T04:50:42,501 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,502 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=CLOSED 2024-11-24T04:50:42,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,46377,1732423825594 because future has completed 2024-11-24T04:50:42,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T04:50:42,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,46377,1732423825594 in 203 msec 2024-11-24T04:50:42,511 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE; state=CLOSED, location=4464c5b832df,36869,1732423825485; forceNewPlan=false, retain=false 2024-11-24T04:50:42,662 INFO [4464c5b832df:36681 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T04:50:42,663 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPENING, regionLocation=4464c5b832df,36869,1732423825485 2024-11-24T04:50:42,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE because future has completed 2024-11-24T04:50:42,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485}] 2024-11-24T04:50:42,824 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,824 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a87c32a42ac0dc0089a1412f6f2216d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:42,824 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,825 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:42,825 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,825 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,826 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,827 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf1 2024-11-24T04:50:42,827 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:42,835 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/bd61afa88bea4267b07e8297b58cbb13 2024-11-24T04:50:42,835 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:42,835 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,837 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf2 2024-11-24T04:50:42,837 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:42,844 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/ac90313313d24a9792cb461bb5cec2c5 2024-11-24T04:50:42,845 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:42,845 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,846 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,847 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,848 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,848 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,849 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:50:42,851 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,852 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 8a87c32a42ac0dc0089a1412f6f2216d; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60478674, jitterRate=-0.09879752993583679}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:50:42,852 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,852 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 8a87c32a42ac0dc0089a1412f6f2216d: Running coprocessor pre-open hook at 1732423842825Writing region info on filesystem at 1732423842825Initializing all the Stores at 1732423842826 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423842826Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423842826Cleaning up temporary data from old regions at 1732423842848 (+22 ms)Running coprocessor post-open hooks at 1732423842852 (+4 ms)Region opened successfully at 1732423842852 2024-11-24T04:50:42,854 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., pid=12, masterSystemTime=1732423842819 2024-11-24T04:50:42,856 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,856 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,857 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPEN, openSeqNum=18, regionLocation=4464c5b832df,36869,1732423825485 2024-11-24T04:50:42,861 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36681 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=4464c5b832df,36869,1732423825485, table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T04:50:42,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485 because future has completed 2024-11-24T04:50:42,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T04:50:42,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,36869,1732423825485 in 197 msec 2024-11-24T04:50:42,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, REOPEN/MOVE in 570 msec 2024-11-24T04:50:42,900 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T04:50:42,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T04:50:42,903 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 4464c5b832df,36869,1732423825485: testing ***** 2024-11-24T04:50:42,903 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T04:50:42,905 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T04:50:42,907 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T04:50:42,911 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T04:50:42,913 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T04:50:42,924 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "Verbose": false, "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 319624936 }, "NonHeapMemoryUsage": { "committed": 171180032, "init": 7667712, "max": -1, "used": 168560000 }, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "4464c5b832df", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2068, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 9, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 8, "ProcessCallTime_95th_percentile": 8, "ProcessCallTime_98th_percentile": 8, "ProcessCallTime_99th_percentile": 8, "ProcessCallTime_99.9th_percentile": 8, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 9, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 8, "TotalCallTime_95th_percentile": 8, "TotalCallTime_98th_percentile": 8, "TotalCallTime_99th_percentile": 8, "TotalCallTime_99.9th_percentile": 8, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 175, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 131, "ResponseSize_90th_percentile": 157, "ResponseSize_95th_percentile": 166, "ResponseSize_98th_percentile": 171, "ResponseSize_99th_percentile": 173, "ResponseSize_99.9th_percentile": 174, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 352 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "4464c5b832df", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:49287", "tag.serverName": "4464c5b832df,42181,1732423825646", "tag.clusterId": "287bef7b-d135-4e75-8479-3603de413d72", "tag.Context": "regionserver", "tag.Hostname": "4464c5b832df", "regionCount": 1, "storeCount": 4, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732423825646, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.4, "writeRequestRatePerSecond": 0.4, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 199680, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 7, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 4, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 4, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 4, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 2, "Put_min": 2, "Put_max": 7, "Put_mean": 4, "Put_25th_percentile": 3, "Put_median": 4, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 2, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 4, "ScanSize_min": 0, "ScanSize_max": 144, "ScanSize_mean": 72, "ScanSize_25th_percentile": 36, "ScanSize_median": 72, "ScanSize_75th_percentile": 108, "ScanSize_90th_percentile": 129, "ScanSize_95th_percentile": 136, "ScanSize_98th_percentile": 141, "ScanSize_99th_percentile": 142, "ScanSize_99.9th_percentile": 143, "ScanSize_SizeRangeCount_0-10": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T04:50:42,927 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36681 {}] master.MasterRpcServices(700): 4464c5b832df,36869,1732423825485 reported a fatal error: ***** ABORTING region server 4464c5b832df,36869,1732423825485: testing ***** 2024-11-24T04:50:42,930 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4464c5b832df,36869,1732423825485' ***** 2024-11-24T04:50:42,930 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T04:50:42,931 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:50:42,931 INFO [RS:0;4464c5b832df:36869 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T04:50:42,931 INFO [RS:0;4464c5b832df:36869 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T04:50:42,931 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,932 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(956): aborting server 4464c5b832df,36869,1732423825485 2024-11-24T04:50:42,932 INFO [RS:0;4464c5b832df:36869 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:50:42,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46377 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:40288 deadline: 1732423902932, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=36869 startCode=1732423825485. As of locationSeqNum=12. 2024-11-24T04:50:42,932 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a87c32a42ac0dc0089a1412f6f2216d, disabling compactions & flushes 2024-11-24T04:50:42,932 INFO [RS:0;4464c5b832df:36869 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4464c5b832df:36869. 2024-11-24T04:50:42,932 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,932 DEBUG [RS:0;4464c5b832df:36869 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:50:42,932 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,932 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. after waiting 0 ms 2024-11-24T04:50:42,932 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,933 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:50:42,933 DEBUG [RS:0;4464c5b832df:36869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:50:42,933 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T04:50:42,933 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1325): Online Regions={8a87c32a42ac0dc0089a1412f6f2216d=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.} 2024-11-24T04:50:42,933 DEBUG [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1351): Waiting on 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:42,934 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,46377,1732423825594, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,46377,1732423825594, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=36869 startCode=1732423825485. As of locationSeqNum=12. 2024-11-24T04:50:42,934 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,46377,1732423825594, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=36869 startCode=1732423825485. As of locationSeqNum=12. 2024-11-24T04:50:42,934 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,46377,1732423825594, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=4464c5b832df port=36869 startCode=1732423825485. As of locationSeqNum=12. 2024-11-24T04:50:42,937 INFO [regionserver/4464c5b832df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:50:42,945 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:42,945 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a87c32a42ac0dc0089a1412f6f2216d: Waiting for close lock at 1732423842932Running coprocessor pre-close hooks at 1732423842932Disabling compacts and flushes for region at 1732423842932Disabling writes for close at 1732423842932Writing region close event to WAL at 1732423842944 (+12 ms)Running coprocessor post-close hooks at 1732423842944Closed at 1732423842945 (+1 ms) 2024-11-24T04:50:42,945 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:43,005 INFO [regionserver/4464c5b832df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T04:50:43,005 INFO [regionserver/4464c5b832df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T04:50:43,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4464c5b832df,36869,1732423825485 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:43,049 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4464c5b832df,36869,1732423825485 aborting 2024-11-24T04:50:43,049 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 4464c5b832df,36869,1732423825485 aborting 2024-11-24T04:50:43,049 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=12 from cache 2024-11-24T04:50:43,134 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(976): stopping server 4464c5b832df,36869,1732423825485; all regions closed. 2024-11-24T04:50:43,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741835_1011 (size=1677) 2024-11-24T04:50:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741835_1011 (size=1677) 2024-11-24T04:50:43,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741835_1011 (size=1677) 2024-11-24T04:50:43,138 DEBUG [RS:0;4464c5b832df:36869 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:50:43,139 INFO [RS:0;4464c5b832df:36869 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:50:43,139 INFO [RS:0;4464c5b832df:36869 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:50:43,139 INFO [RS:0;4464c5b832df:36869 {}] hbase.ChoreService(370): Chore service for: regionserver/4464c5b832df:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:50:43,139 INFO [regionserver/4464c5b832df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:50:43,139 INFO [RS:0;4464c5b832df:36869 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:50:43,139 INFO [RS:0;4464c5b832df:36869 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:50:43,139 INFO [RS:0;4464c5b832df:36869 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:50:43,139 INFO [RS:0;4464c5b832df:36869 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:50:43,140 INFO [RS:0;4464c5b832df:36869 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36869 2024-11-24T04:50:43,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:50:43,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4464c5b832df,36869,1732423825485 2024-11-24T04:50:43,158 INFO [RS:0;4464c5b832df:36869 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:50:43,173 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4464c5b832df,36869,1732423825485] 2024-11-24T04:50:43,183 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4464c5b832df,36869,1732423825485 already deleted, retry=false 2024-11-24T04:50:43,183 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 4464c5b832df,36869,1732423825485 on 4464c5b832df,36681,1732423824726 2024-11-24T04:50:43,188 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 4464c5b832df,36869,1732423825485, splitWal=true, meta=false 2024-11-24T04:50:43,191 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 4464c5b832df,36869,1732423825485 (carryingMeta=false) 4464c5b832df,36869,1732423825485/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@11571106[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T04:50:43,192 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 4464c5b832df,36869,1732423825485, splitWal=true, meta=false 2024-11-24T04:50:43,195 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 4464c5b832df,36869,1732423825485 had 1 regions 2024-11-24T04:50:43,198 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 4464c5b832df,36869,1732423825485, splitWal=true, meta=false, isMeta: false 2024-11-24T04:50:43,199 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting 2024-11-24T04:50:43,200 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4464c5b832df,36869,1732423825485 WAL count=1, meta=false 2024-11-24T04:50:43,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 4464c5b832df%2C36869%2C1732423825485.1732423827167}] 2024-11-24T04:50:43,208 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=4464c5b832df,42181,1732423825646 2024-11-24T04:50:43,212 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 4464c5b832df%2C36869%2C1732423825485.1732423827167, worker=4464c5b832df,42181,1732423825646}] 2024-11-24T04:50:43,260 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18] 2024-11-24T04:50:43,262 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 4464c5b832df:36869 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 4464c5b832df/172.17.0.2:36869 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:43,262 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18, error=java.net.ConnectException: Call to address=4464c5b832df:36869 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 4464c5b832df/172.17.0.2:36869 2024-11-24T04:50:43,262 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-24T04:50:43,262 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18 from cache 2024-11-24T04:50:43,263 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address 4464c5b832df:36869 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 4464c5b832df/172.17.0.2:36869 2024-11-24T04:50:43,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:50:43,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36869-0x1016b2dadec0001, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:50:43,273 INFO [RS:0;4464c5b832df:36869 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:50:43,273 INFO [RS:0;4464c5b832df:36869 {}] regionserver.HRegionServer(1031): Exiting; stopping=4464c5b832df,36869,1732423825485; zookeeper connection closed. 2024-11-24T04:50:43,274 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ae009e9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ae009e9 2024-11-24T04:50:43,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42181 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T04:50:43,396 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167, size=1.6 K (1677bytes) 2024-11-24T04:50:43,396 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167 2024-11-24T04:50:43,396 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167 after 0ms 2024-11-24T04:50:43,405 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:43,405 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167 took 10ms 2024-11-24T04:50:43,418 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 8a87c32a42ac0dc0089a1412f6f2216d: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T04:50:43,418 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167 so closing down 2024-11-24T04:50:43,419 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:43,420 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:43,420 INFO [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167, size=1.6 K, length=1677, corrupted=false, cancelled=false 2024-11-24T04:50:43,420 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167, journal: Splitting hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167, size=1.6 K (1677bytes) at 1732423843396Finishing writing output for hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167 so closing down at 1732423843418 (+22 ms)3 split writer threads finished at 1732423843420 (+2 ms)Processed 6 edits across 0 Regions in 10 ms; skipped=6; WAL=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167, size=1.6 K, length=1677, corrupted=false, cancelled=false at 1732423843420 2024-11-24T04:50:43,420 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167 2024-11-24T04:50:43,422 DEBUG [RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T04:50:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36681 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T04:50:43,431 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting/4464c5b832df%2C36869%2C1732423825485.1732423827167 to hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs 2024-11-24T04:50:43,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T04:50:43,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 4464c5b832df%2C36869%2C1732423825485.1732423827167, worker=4464c5b832df,42181,1732423825646 in 220 msec 2024-11-24T04:50:43,448 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=4464c5b832df,42181,1732423825646 2024-11-24T04:50:43,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T04:50:43,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 4464c5b832df%2C36869%2C1732423825485.1732423827167, worker=4464c5b832df,42181,1732423825646 in 247 msec 2024-11-24T04:50:43,455 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting dir is empty, no logs to split. 2024-11-24T04:50:43,455 INFO [PEWorker-1 {}] master.SplitWALManager(105): 4464c5b832df,36869,1732423825485 WAL count=0, meta=false 2024-11-24T04:50:43,455 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 4464c5b832df,36869,1732423825485 WAL splitting is done? wals=0, meta=false 2024-11-24T04:50:43,458 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 4464c5b832df,36869,1732423825485 failed, ignore...File hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/WALs/4464c5b832df,36869,1732423825485-splitting does not exist. 2024-11-24T04:50:43,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN}] 2024-11-24T04:50:43,462 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN 2024-11-24T04:50:43,464 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T04:50:43,570 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18] 2024-11-24T04:50:43,571 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to 4464c5b832df:36869 this server is in the failed servers list 2024-11-24T04:50:43,571 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=4464c5b832df:36869 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 4464c5b832df:36869 2024-11-24T04:50:43,571 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 4464c5b832df:36869 2024-11-24T04:50:43,571 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,36869,1732423825485, seqNum=18 from cache 2024-11-24T04:50:43,615 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(204): Hosts are {4464c5b832df=0} racks are {/default-rack=0} 2024-11-24T04:50:43,615 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T04:50:43,615 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T04:50:43,615 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T04:50:43,615 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T04:50:43,615 INFO [4464c5b832df:36681 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T04:50:43,615 INFO [4464c5b832df:36681 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T04:50:43,615 DEBUG [4464c5b832df:36681 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T04:50:43,616 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPENING, regionLocation=4464c5b832df,42181,1732423825646 2024-11-24T04:50:43,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN because future has completed 2024-11-24T04:50:43,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,42181,1732423825646}] 2024-11-24T04:50:43,776 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:43,777 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a87c32a42ac0dc0089a1412f6f2216d, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:43,777 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,777 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:43,777 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,777 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,779 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,780 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf1 2024-11-24T04:50:43,780 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:43,786 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf1/bd61afa88bea4267b07e8297b58cbb13 2024-11-24T04:50:43,786 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:43,786 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,787 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a87c32a42ac0dc0089a1412f6f2216d columnFamilyName cf2 2024-11-24T04:50:43,788 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:43,794 DEBUG [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/cf2/ac90313313d24a9792cb461bb5cec2c5 2024-11-24T04:50:43,794 INFO [StoreOpener-8a87c32a42ac0dc0089a1412f6f2216d-1 {}] regionserver.HStore(327): Store=8a87c32a42ac0dc0089a1412f6f2216d/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:43,794 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,795 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,797 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,798 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,798 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,799 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T04:50:43,800 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,802 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 8a87c32a42ac0dc0089a1412f6f2216d; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65196856, jitterRate=-0.02849113941192627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T04:50:43,802 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:50:43,803 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 8a87c32a42ac0dc0089a1412f6f2216d: Running coprocessor pre-open hook at 1732423843777Writing region info on filesystem at 1732423843777Initializing all the Stores at 1732423843778 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423843778Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423843778Cleaning up temporary data from old regions at 1732423843798 (+20 ms)Running coprocessor post-open hooks at 1732423843802 (+4 ms)Region opened successfully at 1732423843803 (+1 ms) 2024-11-24T04:50:43,804 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., pid=17, masterSystemTime=1732423843772 2024-11-24T04:50:43,808 DEBUG [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:43,808 INFO [RS_OPEN_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:50:43,809 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=8a87c32a42ac0dc0089a1412f6f2216d, regionState=OPEN, openSeqNum=18, regionLocation=4464c5b832df,42181,1732423825646 2024-11-24T04:50:43,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,42181,1732423825646 because future has completed 2024-11-24T04:50:43,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T04:50:43,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 8a87c32a42ac0dc0089a1412f6f2216d, server=4464c5b832df,42181,1732423825646 in 197 msec 2024-11-24T04:50:43,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T04:50:43,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=8a87c32a42ac0dc0089a1412f6f2216d, ASSIGN in 360 msec 2024-11-24T04:50:43,823 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server 4464c5b832df,36869,1732423825485 after splitting done 2024-11-24T04:50:43,827 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 4464c5b832df,36869,1732423825485, splitWal=true, meta=false in 639 msec 2024-11-24T04:50:44,080 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d., hostname=4464c5b832df,42181,1732423825646, seqNum=18] 2024-11-24T04:50:44,093 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterRegionMovedWithMultiCF Thread=415 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1417771994_22 at /127.0.0.1:43182 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1693438953_22 at /127.0.0.1:58292 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4464c5b832df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_LOG_REPLAY_OPS-regionserver/4464c5b832df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_397228859_22 at /127.0.0.1:47580 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4464c5b832df:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-6-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 410) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=10694 (was 10836) 2024-11-24T04:50:44,095 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-24T04:50:44,105 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=415, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=10692 2024-11-24T04:50:44,105 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-24T04:50:44,121 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:44,123 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:44,124 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:44,127 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36612302, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-36612302, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:44,140 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36612302/hregion-36612302.1732423844128, exclude list is [], retry=0 2024-11-24T04:50:44,143 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:44,143 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:44,143 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:44,146 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36612302/hregion-36612302.1732423844128 2024-11-24T04:50:44,146 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:44,146 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6c4574dc77c67ef73f2b49261aad0530, NAME => 'testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:44,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:50:44,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:50:44,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741893_1071 (size=67) 2024-11-24T04:50:44,161 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:44,162 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,164 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName a 2024-11-24T04:50:44,164 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,165 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,165 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,166 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName b 2024-11-24T04:50:44,166 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,167 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,167 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,168 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName c 2024-11-24T04:50:44,168 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,169 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,169 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,169 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,170 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,171 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,171 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,171 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:44,172 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,174 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:44,174 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c4574dc77c67ef73f2b49261aad0530; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64415178, jitterRate=-0.04013904929161072}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:44,175 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c4574dc77c67ef73f2b49261aad0530: Writing region info on filesystem at 1732423844161Initializing all the Stores at 1732423844162 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844162Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844162Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844162Cleaning up temporary data from old regions at 1732423844171 (+9 ms)Region opened successfully at 1732423844175 (+4 ms) 2024-11-24T04:50:44,175 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6c4574dc77c67ef73f2b49261aad0530, disabling compactions & flushes 2024-11-24T04:50:44,175 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,175 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,175 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. after waiting 0 ms 2024-11-24T04:50:44,175 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,176 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,176 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6c4574dc77c67ef73f2b49261aad0530: Waiting for close lock at 1732423844175Disabling compacts and flushes for region at 1732423844175Disabling writes for close at 1732423844175Writing region close event to WAL at 1732423844176 (+1 ms)Closed at 1732423844176 2024-11-24T04:50:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741892_1070 (size=93) 2024-11-24T04:50:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741892_1070 (size=93) 2024-11-24T04:50:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741892_1070 (size=93) 2024-11-24T04:50:44,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:44,180 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36612302:(num 1732423844128) 2024-11-24T04:50:44,181 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:44,183 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:44,200 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183, exclude list is [], retry=0 2024-11-24T04:50:44,203 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:44,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:44,204 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:44,206 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183 2024-11-24T04:50:44,206 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:44,206 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c4574dc77c67ef73f2b49261aad0530, NAME => 'testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:44,206 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:44,206 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,207 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,208 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,209 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName a 2024-11-24T04:50:44,209 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,210 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,210 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,211 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName b 2024-11-24T04:50:44,211 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,212 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,212 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,213 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName c 2024-11-24T04:50:44,213 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,213 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,213 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,214 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,217 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,217 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,218 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:44,220 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,221 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c4574dc77c67ef73f2b49261aad0530; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72608426, jitterRate=0.0819498598575592}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:44,222 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c4574dc77c67ef73f2b49261aad0530: Writing region info on filesystem at 1732423844207Initializing all the Stores at 1732423844208 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844208Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844208Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844208Cleaning up temporary data from old regions at 1732423844217 (+9 ms)Region opened successfully at 1732423844222 (+5 ms) 2024-11-24T04:50:44,258 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6c4574dc77c67ef73f2b49261aad0530 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T04:50:44,274 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/a/65a10a24f6444cd184036c992cbf2709 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732423844222/Put/seqid=0 2024-11-24T04:50:44,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:50:44,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:50:44,287 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/a/65a10a24f6444cd184036c992cbf2709 2024-11-24T04:50:44,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741895_1073 (size=5958) 2024-11-24T04:50:44,309 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/b/38aeaa591cf44015ba4436dbc7dd8c5e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732423844234/Put/seqid=0 2024-11-24T04:50:44,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:50:44,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:50:44,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741896_1074 (size=5958) 2024-11-24T04:50:44,325 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/b/38aeaa591cf44015ba4436dbc7dd8c5e 2024-11-24T04:50:44,345 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/c/c764900dba9b4062bae26f7fed5dae2c is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732423844244/Put/seqid=0 2024-11-24T04:50:44,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:50:44,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:50:44,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741897_1075 (size=5958) 2024-11-24T04:50:44,353 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/c/c764900dba9b4062bae26f7fed5dae2c 2024-11-24T04:50:44,360 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/a/65a10a24f6444cd184036c992cbf2709 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/a/65a10a24f6444cd184036c992cbf2709 2024-11-24T04:50:44,371 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/a/65a10a24f6444cd184036c992cbf2709, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:50:44,373 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/b/38aeaa591cf44015ba4436dbc7dd8c5e as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/b/38aeaa591cf44015ba4436dbc7dd8c5e 2024-11-24T04:50:44,378 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/b/38aeaa591cf44015ba4436dbc7dd8c5e, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:50:44,379 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/c/c764900dba9b4062bae26f7fed5dae2c as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/c/c764900dba9b4062bae26f7fed5dae2c 2024-11-24T04:50:44,385 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/c/c764900dba9b4062bae26f7fed5dae2c, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T04:50:44,387 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 6c4574dc77c67ef73f2b49261aad0530 in 128ms, sequenceid=33, compaction requested=false 2024-11-24T04:50:44,387 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6c4574dc77c67ef73f2b49261aad0530: 2024-11-24T04:50:44,387 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6c4574dc77c67ef73f2b49261aad0530, disabling compactions & flushes 2024-11-24T04:50:44,387 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,387 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,387 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. after waiting 0 ms 2024-11-24T04:50:44,387 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,396 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. 2024-11-24T04:50:44,396 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6c4574dc77c67ef73f2b49261aad0530: Waiting for close lock at 1732423844387Disabling compacts and flushes for region at 1732423844387Disabling writes for close at 1732423844387Writing region close event to WAL at 1732423844395 (+8 ms)Closed at 1732423844395 2024-11-24T04:50:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741894_1072 (size=2873) 2024-11-24T04:50:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741894_1072 (size=2873) 2024-11-24T04:50:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741894_1072 (size=2873) 2024-11-24T04:50:44,405 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/b/38aeaa591cf44015ba4436dbc7dd8c5e to hdfs://localhost:44683/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/b/38aeaa591cf44015ba4436dbc7dd8c5e 2024-11-24T04:50:44,424 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183, size=2.8 K (2873bytes) 2024-11-24T04:50:44,424 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183 2024-11-24T04:50:44,425 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183 after 1ms 2024-11-24T04:50:44,428 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:44,429 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183 took 5ms 2024-11-24T04:50:44,432 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183 so closing down 2024-11-24T04:50:44,432 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:44,433 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423844183.temp 2024-11-24T04:50:44,435 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000003-wal.1732423844183.temp 2024-11-24T04:50:44,435 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:44,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741898_1076 (size=2312) 2024-11-24T04:50:44,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741898_1076 (size=2312) 2024-11-24T04:50:44,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741898_1076 (size=2312) 2024-11-24T04:50:44,448 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000003-wal.1732423844183.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:44,450 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000003-wal.1732423844183.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000032 2024-11-24T04:50:44,450 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183, size=2.8 K, length=2873, corrupted=false, cancelled=false 2024-11-24T04:50:44,450 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183, journal: Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183, size=2.8 K (2873bytes) at 1732423844424Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183 so closing down at 1732423844432 (+8 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000003-wal.1732423844183.temp at 1732423844435 (+3 ms)3 split writer threads finished at 1732423844435Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000003-wal.1732423844183.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732423844448 (+13 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000003-wal.1732423844183.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000032 at 1732423844450 (+2 ms)Processed 32 edits across 1 Regions in 21 ms; skipped=2; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183, size=2.8 K, length=2873, corrupted=false, cancelled=false at 1732423844450 2024-11-24T04:50:44,452 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844183 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423844183 2024-11-24T04:50:44,453 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000032 2024-11-24T04:50:44,453 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:44,455 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:44,474 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844456, exclude list is [], retry=0 2024-11-24T04:50:44,477 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:44,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:44,478 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:44,480 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844456 2024-11-24T04:50:44,482 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:44,482 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c4574dc77c67ef73f2b49261aad0530, NAME => 'testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:44,482 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:44,482 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,482 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,484 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,484 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName a 2024-11-24T04:50:44,484 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,490 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/a/65a10a24f6444cd184036c992cbf2709 2024-11-24T04:50:44,491 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,491 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,492 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName b 2024-11-24T04:50:44,493 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,493 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,493 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,494 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c4574dc77c67ef73f2b49261aad0530 columnFamilyName c 2024-11-24T04:50:44,494 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,500 DEBUG [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/c/c764900dba9b4062bae26f7fed5dae2c 2024-11-24T04:50:44,500 INFO [StoreOpener-6c4574dc77c67ef73f2b49261aad0530-1 {}] regionserver.HStore(327): Store=6c4574dc77c67ef73f2b49261aad0530/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,500 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,503 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,503 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000032 2024-11-24T04:50:44,506 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:44,508 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000032 2024-11-24T04:50:44,508 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6c4574dc77c67ef73f2b49261aad0530 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T04:50:44,524 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/b/f224dc311a3a43dba11e084e07b6e607 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732423844234/Put/seqid=0 2024-11-24T04:50:44,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:50:44,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:50:44,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741900_1078 (size=5958) 2024-11-24T04:50:44,530 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/b/f224dc311a3a43dba11e084e07b6e607 2024-11-24T04:50:44,537 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/.tmp/b/f224dc311a3a43dba11e084e07b6e607 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/b/f224dc311a3a43dba11e084e07b6e607 2024-11-24T04:50:44,543 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/b/f224dc311a3a43dba11e084e07b6e607, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T04:50:44,543 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 6c4574dc77c67ef73f2b49261aad0530 in 35ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T04:50:44,544 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/0000000000000000032 2024-11-24T04:50:44,545 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,545 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,546 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:44,548 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6c4574dc77c67ef73f2b49261aad0530 2024-11-24T04:50:44,550 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/6c4574dc77c67ef73f2b49261aad0530/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T04:50:44,551 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6c4574dc77c67ef73f2b49261aad0530; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60339810, jitterRate=-0.10086676478385925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:44,552 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6c4574dc77c67ef73f2b49261aad0530: Writing region info on filesystem at 1732423844482Initializing all the Stores at 1732423844483 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844483Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844483Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844483Obtaining lock to block concurrent updates at 1732423844508 (+25 ms)Preparing flush snapshotting stores in 6c4574dc77c67ef73f2b49261aad0530 at 1732423844508Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732423844508Flushing stores of testReplayEditsWrittenViaHRegion,,1732423844122.6c4574dc77c67ef73f2b49261aad0530. at 1732423844508Flushing 6c4574dc77c67ef73f2b49261aad0530/b: creating writer at 1732423844508Flushing 6c4574dc77c67ef73f2b49261aad0530/b: appending metadata at 1732423844523 (+15 ms)Flushing 6c4574dc77c67ef73f2b49261aad0530/b: closing flushed file at 1732423844523Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57b2d996: reopening flushed file at 1732423844536 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 6c4574dc77c67ef73f2b49261aad0530 in 35ms, sequenceid=32, compaction requested=false; wal=null at 1732423844544 (+8 ms)Cleaning up temporary data from old regions at 1732423844545 (+1 ms)Region opened successfully at 1732423844552 (+7 ms) 2024-11-24T04:50:44,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterPartialFlush Thread=424 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43182 [Waiting for operation #31] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:46042 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:58292 [Waiting for operation #37] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:47580 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:43288 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51238 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=417 (was 417), ProcessCount=11 (was 11), AvailableMemoryMB=10678 (was 10692) 2024-11-24T04:50:44,573 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-24T04:50:44,587 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=424, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=417, ProcessCount=11, AvailableMemoryMB=10676 2024-11-24T04:50:44,588 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-24T04:50:44,604 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:44,606 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:44,607 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:44,614 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89208680, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-89208680, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:44,632 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89208680/hregion-89208680.1732423844614, exclude list is [], retry=0 2024-11-24T04:50:44,641 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:44,644 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:44,646 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:44,660 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89208680/hregion-89208680.1732423844614 2024-11-24T04:50:44,661 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:44,662 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3aeeb8081200c66a56f0af4ce66176a1, NAME => 'testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:44,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:50:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:50:44,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741902_1080 (size=68) 2024-11-24T04:50:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:44,684 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,686 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName a 2024-11-24T04:50:44,686 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,686 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,686 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,688 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName b 2024-11-24T04:50:44,688 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,689 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,689 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,691 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName c 2024-11-24T04:50:44,691 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,692 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,693 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,693 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,694 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,695 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,695 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,695 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:44,697 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,699 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:44,700 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3aeeb8081200c66a56f0af4ce66176a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74310067, jitterRate=0.10730628669261932}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:44,701 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3aeeb8081200c66a56f0af4ce66176a1: Writing region info on filesystem at 1732423844683Initializing all the Stores at 1732423844684 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844684Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844684Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844684Cleaning up temporary data from old regions at 1732423844695 (+11 ms)Region opened successfully at 1732423844701 (+6 ms) 2024-11-24T04:50:44,701 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3aeeb8081200c66a56f0af4ce66176a1, disabling compactions & flushes 2024-11-24T04:50:44,701 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:44,701 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:44,701 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. after waiting 0 ms 2024-11-24T04:50:44,701 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:44,702 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:44,702 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3aeeb8081200c66a56f0af4ce66176a1: Waiting for close lock at 1732423844701Disabling compacts and flushes for region at 1732423844701Disabling writes for close at 1732423844701Writing region close event to WAL at 1732423844701Closed at 1732423844701 2024-11-24T04:50:44,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741901_1079 (size=93) 2024-11-24T04:50:44,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741901_1079 (size=93) 2024-11-24T04:50:44,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741901_1079 (size=93) 2024-11-24T04:50:44,709 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:44,709 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89208680:(num 1732423844614) 2024-11-24T04:50:44,709 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:44,711 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:44,723 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711, exclude list is [], retry=0 2024-11-24T04:50:44,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:44,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:44,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:44,728 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711 2024-11-24T04:50:44,728 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:44,812 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3aeeb8081200c66a56f0af4ce66176a1, NAME => 'testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:44,816 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,816 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:44,816 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,816 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,825 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,827 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName a 2024-11-24T04:50:44,827 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,833 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,833 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,834 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName b 2024-11-24T04:50:44,834 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,834 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,835 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,836 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName c 2024-11-24T04:50:44,836 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:44,836 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:44,836 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,837 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,838 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,839 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,839 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,840 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:44,842 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,843 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3aeeb8081200c66a56f0af4ce66176a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70122225, jitterRate=0.04490257799625397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:44,843 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:44,844 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3aeeb8081200c66a56f0af4ce66176a1: Running coprocessor pre-open hook at 1732423844817Writing region info on filesystem at 1732423844817Initializing all the Stores at 1732423844819 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844819Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844824 (+5 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423844824Cleaning up temporary data from old regions at 1732423844839 (+15 ms)Running coprocessor post-open hooks at 1732423844843 (+4 ms)Region opened successfully at 1732423844844 (+1 ms) 2024-11-24T04:50:44,862 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3aeeb8081200c66a56f0af4ce66176a1 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T04:50:44,864 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:45,062 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T04:50:45,062 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:45,063 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T04:50:45,064 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:45,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T04:50:45,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:45,066 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T04:50:45,066 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:45,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T04:50:45,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:45,865 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:46,580 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:50:46,866 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:47,866 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:48,867 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:49,868 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:50,868 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:51,869 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:52,870 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:53,870 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 3aeeb8081200c66a56f0af4ce66176a1/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:53,872 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3aeeb8081200c66a56f0af4ce66176a1: 2024-11-24T04:50:53,872 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:53,887 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T04:50:53,889 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3aeeb8081200c66a56f0af4ce66176a1: 2024-11-24T04:50:53,889 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T04:50:53,889 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3aeeb8081200c66a56f0af4ce66176a1, disabling compactions & flushes 2024-11-24T04:50:53,889 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:53,889 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:53,889 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. after waiting 0 ms 2024-11-24T04:50:53,889 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:53,890 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:53,890 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. 2024-11-24T04:50:53,890 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3aeeb8081200c66a56f0af4ce66176a1: Waiting for close lock at 1732423853889Running coprocessor pre-close hooks at 1732423853889Disabling compacts and flushes for region at 1732423853889Disabling writes for close at 1732423853889Writing region close event to WAL at 1732423853890 (+1 ms)Running coprocessor post-close hooks at 1732423853890Closed at 1732423853890 2024-11-24T04:50:53,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741903_1081 (size=2398) 2024-11-24T04:50:53,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741903_1081 (size=2398) 2024-11-24T04:50:53,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741903_1081 (size=2398) 2024-11-24T04:50:53,909 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711, size=2.3 K (2398bytes) 2024-11-24T04:50:53,909 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711 2024-11-24T04:50:53,909 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711 after 0ms 2024-11-24T04:50:53,912 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:53,912 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711 took 4ms 2024-11-24T04:50:53,915 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711 so closing down 2024-11-24T04:50:53,915 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:53,916 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732423844711.temp 2024-11-24T04:50:53,918 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000004-wal.1732423844711.temp 2024-11-24T04:50:53,918 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741904_1082 (size=1672) 2024-11-24T04:50:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741904_1082 (size=1672) 2024-11-24T04:50:53,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741904_1082 (size=1672) 2024-11-24T04:50:53,928 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000004-wal.1732423844711.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:53,929 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000004-wal.1732423844711.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000026 2024-11-24T04:50:53,929 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711, size=2.3 K, length=2398, corrupted=false, cancelled=false 2024-11-24T04:50:53,929 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711, journal: Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711, size=2.3 K (2398bytes) at 1732423853909Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711 so closing down at 1732423853915 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000004-wal.1732423844711.temp at 1732423853918 (+3 ms)3 split writer threads finished at 1732423853918Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000004-wal.1732423844711.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732423853928 (+10 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000004-wal.1732423844711.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000026 at 1732423853929 (+1 ms)Processed 23 edits across 1 Regions in 16 ms; skipped=3; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711, size=2.3 K, length=2398, corrupted=false, cancelled=false at 1732423853929 2024-11-24T04:50:53,931 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423844711 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423844711 2024-11-24T04:50:53,932 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000026 2024-11-24T04:50:53,932 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:53,935 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:53,950 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423853935, exclude list is [], retry=0 2024-11-24T04:50:53,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:53,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:53,953 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:53,955 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423853935 2024-11-24T04:50:53,955 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:53,956 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3aeeb8081200c66a56f0af4ce66176a1, NAME => 'testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:53,956 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,956 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:53,956 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,956 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,958 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,959 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName a 2024-11-24T04:50:53,960 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:53,960 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:53,960 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,961 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName b 2024-11-24T04:50:53,961 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:53,962 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:53,962 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,963 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3aeeb8081200c66a56f0af4ce66176a1 columnFamilyName c 2024-11-24T04:50:53,963 DEBUG [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:53,964 INFO [StoreOpener-3aeeb8081200c66a56f0af4ce66176a1-1 {}] regionserver.HStore(327): Store=3aeeb8081200c66a56f0af4ce66176a1/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:53,965 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,966 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,967 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:53,968 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000026 2024-11-24T04:50:53,971 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:53,973 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000026 2024-11-24T04:50:53,974 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3aeeb8081200c66a56f0af4ce66176a1 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T04:50:53,995 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/a/1114afb1217c4705bd2f7cf46eeadcc8 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732423853879/Put/seqid=0 2024-11-24T04:50:54,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:50:54,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:50:54,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741906_1084 (size=5523) 2024-11-24T04:50:54,003 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/a/1114afb1217c4705bd2f7cf46eeadcc8 2024-11-24T04:50:54,029 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/b/2cebf43f5aed48c0960acb94eebe9adb is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732423853872/Put/seqid=0 2024-11-24T04:50:54,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:50:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:50:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741907_1085 (size=5524) 2024-11-24T04:50:54,043 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/b/2cebf43f5aed48c0960acb94eebe9adb 2024-11-24T04:50:54,065 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/c/9fd963696edb46e58e5989ecb80f9ac1 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732423853876/Put/seqid=0 2024-11-24T04:50:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:50:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:50:54,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741908_1086 (size=5457) 2024-11-24T04:50:54,073 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/c/9fd963696edb46e58e5989ecb80f9ac1 2024-11-24T04:50:54,079 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/a/1114afb1217c4705bd2f7cf46eeadcc8 as hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/a/1114afb1217c4705bd2f7cf46eeadcc8 2024-11-24T04:50:54,087 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/a/1114afb1217c4705bd2f7cf46eeadcc8, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T04:50:54,089 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/b/2cebf43f5aed48c0960acb94eebe9adb as hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/b/2cebf43f5aed48c0960acb94eebe9adb 2024-11-24T04:50:54,095 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/b/2cebf43f5aed48c0960acb94eebe9adb, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T04:50:54,096 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/.tmp/c/9fd963696edb46e58e5989ecb80f9ac1 as hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/c/9fd963696edb46e58e5989ecb80f9ac1 2024-11-24T04:50:54,102 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/c/9fd963696edb46e58e5989ecb80f9ac1, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T04:50:54,102 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 3aeeb8081200c66a56f0af4ce66176a1 in 129ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T04:50:54,103 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/0000000000000000026 2024-11-24T04:50:54,104 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:54,104 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:54,105 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:54,106 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:54,109 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsAfterAbortingFlush/3aeeb8081200c66a56f0af4ce66176a1/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T04:50:54,110 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3aeeb8081200c66a56f0af4ce66176a1; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72581873, jitterRate=0.08155418932437897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:54,110 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3aeeb8081200c66a56f0af4ce66176a1 2024-11-24T04:50:54,110 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3aeeb8081200c66a56f0af4ce66176a1: Running coprocessor pre-open hook at 1732423853956Writing region info on filesystem at 1732423853956Initializing all the Stores at 1732423853957 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423853957Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423853958 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423853958Obtaining lock to block concurrent updates at 1732423853974 (+16 ms)Preparing flush snapshotting stores in 3aeeb8081200c66a56f0af4ce66176a1 at 1732423853974Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732423853974Flushing stores of testReplayEditsAfterAbortingFlush,,1732423844605.3aeeb8081200c66a56f0af4ce66176a1. at 1732423853974Flushing 3aeeb8081200c66a56f0af4ce66176a1/a: creating writer at 1732423853974Flushing 3aeeb8081200c66a56f0af4ce66176a1/a: appending metadata at 1732423853994 (+20 ms)Flushing 3aeeb8081200c66a56f0af4ce66176a1/a: closing flushed file at 1732423853995 (+1 ms)Flushing 3aeeb8081200c66a56f0af4ce66176a1/b: creating writer at 1732423854010 (+15 ms)Flushing 3aeeb8081200c66a56f0af4ce66176a1/b: appending metadata at 1732423854028 (+18 ms)Flushing 3aeeb8081200c66a56f0af4ce66176a1/b: closing flushed file at 1732423854029 (+1 ms)Flushing 3aeeb8081200c66a56f0af4ce66176a1/c: creating writer at 1732423854048 (+19 ms)Flushing 3aeeb8081200c66a56f0af4ce66176a1/c: appending metadata at 1732423854065 (+17 ms)Flushing 3aeeb8081200c66a56f0af4ce66176a1/c: closing flushed file at 1732423854065Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c67c3b: reopening flushed file at 1732423854078 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fad07c: reopening flushed file at 1732423854087 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68749dd0: reopening flushed file at 1732423854095 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 3aeeb8081200c66a56f0af4ce66176a1 in 129ms, sequenceid=26, compaction requested=false; wal=null at 1732423854102 (+7 ms)Cleaning up temporary data from old regions at 1732423854104 (+2 ms)Running coprocessor post-open hooks at 1732423854110 (+6 ms)Region opened successfully at 1732423854110 2024-11-24T04:50:54,131 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsAfterAbortingFlush Thread=421 (was 424), OpenFileDescriptor=1183 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 417), ProcessCount=11 (was 11), AvailableMemoryMB=10465 (was 10676) 2024-11-24T04:50:54,131 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-24T04:50:54,144 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=421, OpenFileDescriptor=1183, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=10464 2024-11-24T04:50:54,144 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1183 is superior to 1024 2024-11-24T04:50:54,163 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:54,165 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:54,166 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:54,168 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-30465945, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-30465945, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:54,181 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-30465945/hregion-30465945.1732423854169, exclude list is [], retry=0 2024-11-24T04:50:54,184 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:54,184 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:54,184 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:54,186 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-30465945/hregion-30465945.1732423854169 2024-11-24T04:50:54,187 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:54,187 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f3e528c3e40cef6abcb2e7d555a1f1fb, NAME => 'testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:54,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:50:54,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:50:54,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741910_1088 (size=61) 2024-11-24T04:50:54,196 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:54,198 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,199 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3e528c3e40cef6abcb2e7d555a1f1fb columnFamilyName a 2024-11-24T04:50:54,199 DEBUG [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,199 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(327): Store=f3e528c3e40cef6abcb2e7d555a1f1fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,200 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,200 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,201 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,201 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,202 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,204 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:54,205 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3e528c3e40cef6abcb2e7d555a1f1fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67807487, jitterRate=0.010410293936729431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:54,205 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3e528c3e40cef6abcb2e7d555a1f1fb: Writing region info on filesystem at 1732423854196Initializing all the Stores at 1732423854197 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854197Cleaning up temporary data from old regions at 1732423854201 (+4 ms)Region opened successfully at 1732423854205 (+4 ms) 2024-11-24T04:50:54,205 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f3e528c3e40cef6abcb2e7d555a1f1fb, disabling compactions & flushes 2024-11-24T04:50:54,206 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,206 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,206 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. after waiting 0 ms 2024-11-24T04:50:54,206 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,206 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,206 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f3e528c3e40cef6abcb2e7d555a1f1fb: Waiting for close lock at 1732423854205Disabling compacts and flushes for region at 1732423854205Disabling writes for close at 1732423854206 (+1 ms)Writing region close event to WAL at 1732423854206Closed at 1732423854206 2024-11-24T04:50:54,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741909_1087 (size=93) 2024-11-24T04:50:54,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741909_1087 (size=93) 2024-11-24T04:50:54,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741909_1087 (size=93) 2024-11-24T04:50:54,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:54,212 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-30465945:(num 1732423854169) 2024-11-24T04:50:54,213 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:54,214 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:54,230 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215, exclude list is [], retry=0 2024-11-24T04:50:54,232 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:54,233 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:54,233 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:54,234 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215 2024-11-24T04:50:54,235 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:54,235 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f3e528c3e40cef6abcb2e7d555a1f1fb, NAME => 'testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:54,235 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:54,235 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,235 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,236 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,237 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3e528c3e40cef6abcb2e7d555a1f1fb columnFamilyName a 2024-11-24T04:50:54,237 DEBUG [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,237 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(327): Store=f3e528c3e40cef6abcb2e7d555a1f1fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,238 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,238 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,239 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,239 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,239 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,241 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,242 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3e528c3e40cef6abcb2e7d555a1f1fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71229880, jitterRate=0.06140792369842529}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:54,243 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3e528c3e40cef6abcb2e7d555a1f1fb: Writing region info on filesystem at 1732423854235Initializing all the Stores at 1732423854236 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854236Cleaning up temporary data from old regions at 1732423854239 (+3 ms)Region opened successfully at 1732423854243 (+4 ms) 2024-11-24T04:50:54,250 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f3e528c3e40cef6abcb2e7d555a1f1fb, disabling compactions & flushes 2024-11-24T04:50:54,250 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,250 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,250 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. after waiting 0 ms 2024-11-24T04:50:54,250 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,251 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,251 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. 2024-11-24T04:50:54,251 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f3e528c3e40cef6abcb2e7d555a1f1fb: Waiting for close lock at 1732423854250Disabling compacts and flushes for region at 1732423854250Disabling writes for close at 1732423854250Writing region close event to WAL at 1732423854251 (+1 ms)Closed at 1732423854251 2024-11-24T04:50:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741911_1089 (size=838) 2024-11-24T04:50:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741911_1089 (size=838) 2024-11-24T04:50:54,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741911_1089 (size=838) 2024-11-24T04:50:54,269 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215, size=838 (838bytes) 2024-11-24T04:50:54,269 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215 2024-11-24T04:50:54,270 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215 after 1ms 2024-11-24T04:50:54,272 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:54,272 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215 took 3ms 2024-11-24T04:50:54,274 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215 so closing down 2024-11-24T04:50:54,274 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:54,275 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423854215.temp 2024-11-24T04:50:54,276 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000003-wal.1732423854215.temp 2024-11-24T04:50:54,276 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:54,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741912_1090 (size=838) 2024-11-24T04:50:54,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741912_1090 (size=838) 2024-11-24T04:50:54,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741912_1090 (size=838) 2024-11-24T04:50:54,283 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000003-wal.1732423854215.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:54,284 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000003-wal.1732423854215.temp to hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012 2024-11-24T04:50:54,285 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215, size=838, length=838, corrupted=false, cancelled=false 2024-11-24T04:50:54,285 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215, journal: Splitting hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215, size=838 (838bytes) at 1732423854269Finishing writing output for hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215 so closing down at 1732423854274 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000003-wal.1732423854215.temp at 1732423854276 (+2 ms)3 split writer threads finished at 1732423854276Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000003-wal.1732423854215.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732423854283 (+7 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000003-wal.1732423854215.temp to hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012 at 1732423854284 (+1 ms)Processed 10 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215, size=838, length=838, corrupted=false, cancelled=false at 1732423854285 (+1 ms) 2024-11-24T04:50:54,287 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854215 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423854215 2024-11-24T04:50:54,288 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012 2024-11-24T04:50:54,295 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:54,689 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:54,691 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:54,703 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854691, exclude list is [], retry=0 2024-11-24T04:50:54,705 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:54,706 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:54,706 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:54,707 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854691 2024-11-24T04:50:54,707 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:54,707 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f3e528c3e40cef6abcb2e7d555a1f1fb, NAME => 'testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:54,707 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:54,707 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,708 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,710 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,712 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3e528c3e40cef6abcb2e7d555a1f1fb columnFamilyName a 2024-11-24T04:50:54,712 DEBUG [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,713 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(327): Store=f3e528c3e40cef6abcb2e7d555a1f1fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,713 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,715 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,716 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012 2024-11-24T04:50:54,718 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:54,719 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012 2024-11-24T04:50:54,719 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f3e528c3e40cef6abcb2e7d555a1f1fb 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T04:50:54,740 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/.tmp/a/51f63939a10047229c71c3bc8630f736 is 79, key is testDatalossWhenInputError/a:x0/1732423854243/Put/seqid=0 2024-11-24T04:50:54,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:50:54,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:50:54,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741914_1092 (size=5808) 2024-11-24T04:50:54,752 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/.tmp/a/51f63939a10047229c71c3bc8630f736 2024-11-24T04:50:54,760 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/.tmp/a/51f63939a10047229c71c3bc8630f736 as hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/a/51f63939a10047229c71c3bc8630f736 2024-11-24T04:50:54,769 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/a/51f63939a10047229c71c3bc8630f736, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T04:50:54,769 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f3e528c3e40cef6abcb2e7d555a1f1fb in 50ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T04:50:54,770 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/0000000000000000012 2024-11-24T04:50:54,771 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,771 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,774 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,776 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T04:50:54,777 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3e528c3e40cef6abcb2e7d555a1f1fb; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72718390, jitterRate=0.08358845114707947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:54,778 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3e528c3e40cef6abcb2e7d555a1f1fb: Writing region info on filesystem at 1732423854708Initializing all the Stores at 1732423854710 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854710Obtaining lock to block concurrent updates at 1732423854719 (+9 ms)Preparing flush snapshotting stores in f3e528c3e40cef6abcb2e7d555a1f1fb at 1732423854719Finished memstore snapshotting testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732423854719Flushing stores of testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb. at 1732423854719Flushing f3e528c3e40cef6abcb2e7d555a1f1fb/a: creating writer at 1732423854719Flushing f3e528c3e40cef6abcb2e7d555a1f1fb/a: appending metadata at 1732423854739 (+20 ms)Flushing f3e528c3e40cef6abcb2e7d555a1f1fb/a: closing flushed file at 1732423854739Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c77d5f4: reopening flushed file at 1732423854759 (+20 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f3e528c3e40cef6abcb2e7d555a1f1fb in 50ms, sequenceid=12, compaction requested=false; wal=null at 1732423854769 (+10 ms)Cleaning up temporary data from old regions at 1732423854771 (+2 ms)Region opened successfully at 1732423854778 (+7 ms) 2024-11-24T04:50:54,781 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f3e528c3e40cef6abcb2e7d555a1f1fb, NAME => 'testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:54,782 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732423854163.f3e528c3e40cef6abcb2e7d555a1f1fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:54,782 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,782 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,783 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,784 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3e528c3e40cef6abcb2e7d555a1f1fb columnFamilyName a 2024-11-24T04:50:54,784 DEBUG [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,789 DEBUG [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/a/51f63939a10047229c71c3bc8630f736 2024-11-24T04:50:54,789 INFO [StoreOpener-f3e528c3e40cef6abcb2e7d555a1f1fb-1 {}] regionserver.HStore(327): Store=f3e528c3e40cef6abcb2e7d555a1f1fb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,789 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,790 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,791 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,792 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,792 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,793 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f3e528c3e40cef6abcb2e7d555a1f1fb 2024-11-24T04:50:54,796 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testDatalossWhenInputError/f3e528c3e40cef6abcb2e7d555a1f1fb/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T04:50:54,797 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f3e528c3e40cef6abcb2e7d555a1f1fb; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73254081, jitterRate=0.09157086908817291}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T04:50:54,797 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f3e528c3e40cef6abcb2e7d555a1f1fb: Writing region info on filesystem at 1732423854782Initializing all the Stores at 1732423854783 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854783Cleaning up temporary data from old regions at 1732423854792 (+9 ms)Region opened successfully at 1732423854797 (+5 ms) 2024-11-24T04:50:54,813 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testDatalossWhenInputError Thread=431 (was 421) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:56520 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51468 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:53876 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:56582 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51552 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:53776 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1774581998-172.17.0.2-1732423819869:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1183) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=10614 (was 10464) - AvailableMemoryMB LEAK? - 2024-11-24T04:50:54,813 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T04:50:54,825 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=431, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=10613 2024-11-24T04:50:54,825 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T04:50:54,838 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:54,840 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:54,841 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:54,843 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54841382, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-54841382, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:54,855 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54841382/hregion-54841382.1732423854843, exclude list is [], retry=0 2024-11-24T04:50:54,858 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:54,859 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:54,859 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:54,861 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54841382/hregion-54841382.1732423854843 2024-11-24T04:50:54,861 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:54,862 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6067f4a5f0c0f6bc1ff68dd504605759, NAME => 'testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:54,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:50:54,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:50:54,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741916_1094 (size=63) 2024-11-24T04:50:54,871 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:54,872 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,874 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName a 2024-11-24T04:50:54,874 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,874 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,874 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,876 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName b 2024-11-24T04:50:54,876 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,876 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,876 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,878 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName c 2024-11-24T04:50:54,878 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,878 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,878 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,879 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,879 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,880 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,880 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,881 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:54,882 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,884 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:54,885 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6067f4a5f0c0f6bc1ff68dd504605759; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70745712, jitterRate=0.05419325828552246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6067f4a5f0c0f6bc1ff68dd504605759: Writing region info on filesystem at 1732423854871Initializing all the Stores at 1732423854872 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854872Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854872Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854872Cleaning up temporary data from old regions at 1732423854880 (+8 ms)Region opened successfully at 1732423854886 (+6 ms) 2024-11-24T04:50:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6067f4a5f0c0f6bc1ff68dd504605759, disabling compactions & flushes 2024-11-24T04:50:54,886 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. after waiting 0 ms 2024-11-24T04:50:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:54,887 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:54,887 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6067f4a5f0c0f6bc1ff68dd504605759: Waiting for close lock at 1732423854886Disabling compacts and flushes for region at 1732423854886Disabling writes for close at 1732423854886Writing region close event to WAL at 1732423854886Closed at 1732423854886 2024-11-24T04:50:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741915_1093 (size=93) 2024-11-24T04:50:54,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741915_1093 (size=93) 2024-11-24T04:50:54,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741915_1093 (size=93) 2024-11-24T04:50:54,894 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:54,894 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54841382:(num 1732423854843) 2024-11-24T04:50:54,894 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:54,896 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:54,913 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896, exclude list is [], retry=0 2024-11-24T04:50:54,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:54,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:54,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:54,918 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 2024-11-24T04:50:54,918 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661)] 2024-11-24T04:50:54,918 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6067f4a5f0c0f6bc1ff68dd504605759, NAME => 'testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:54,918 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:54,919 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,919 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,920 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,920 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName a 2024-11-24T04:50:54,920 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,921 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,921 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,922 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName b 2024-11-24T04:50:54,922 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,922 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,922 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,923 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName c 2024-11-24T04:50:54,923 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:54,924 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:54,924 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,925 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,926 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,926 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,927 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,927 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:54,928 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:54,929 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6067f4a5f0c0f6bc1ff68dd504605759; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71036425, jitterRate=0.058525219559669495}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:54,930 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6067f4a5f0c0f6bc1ff68dd504605759: Writing region info on filesystem at 1732423854919Initializing all the Stores at 1732423854919Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854919Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854920 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423854920Cleaning up temporary data from old regions at 1732423854927 (+7 ms)Region opened successfully at 1732423854930 (+3 ms) 2024-11-24T04:50:54,936 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732423854936/Put/seqid=0 2024-11-24T04:50:54,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:50:54,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:50:54,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741918_1096 (size=4875) 2024-11-24T04:50:54,954 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732423854954/Put/seqid=0 2024-11-24T04:50:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:50:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:50:54,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741919_1097 (size=4875) 2024-11-24T04:50:54,964 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732423854964/Put/seqid=0 2024-11-24T04:50:54,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:50:54,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:50:54,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741920_1098 (size=4875) 2024-11-24T04:50:54,977 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 6067f4a5f0c0f6bc1ff68dd504605759/a 2024-11-24T04:50:54,980 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T04:50:54,980 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:50:54,980 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 6067f4a5f0c0f6bc1ff68dd504605759/a 2024-11-24T04:50:54,983 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T04:50:54,984 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:50:54,984 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 6067f4a5f0c0f6bc1ff68dd504605759/a 2024-11-24T04:50:54,987 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T04:50:54,987 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T04:50:54,987 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6067f4a5f0c0f6bc1ff68dd504605759 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T04:50:55,002 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/.tmp/a/ce1ea7583d784049958ed51adb5c41bb is 55, key is testCompactedBulkLoadedFiles/a:a/1732423854930/Put/seqid=0 2024-11-24T04:50:55,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:50:55,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:50:55,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741921_1099 (size=5107) 2024-11-24T04:50:55,009 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/.tmp/a/ce1ea7583d784049958ed51adb5c41bb 2024-11-24T04:50:55,014 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/.tmp/a/ce1ea7583d784049958ed51adb5c41bb as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb 2024-11-24T04:50:55,019 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T04:50:55,020 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 6067f4a5f0c0f6bc1ff68dd504605759 in 33ms, sequenceid=4, compaction requested=false 2024-11-24T04:50:55,020 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6067f4a5f0c0f6bc1ff68dd504605759: 2024-11-24T04:50:55,021 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_ 2024-11-24T04:50:55,022 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_ 2024-11-24T04:50:55,023 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_ 2024-11-24T04:50:55,024 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile0 into 6067f4a5f0c0f6bc1ff68dd504605759/a as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_ - updating store file list. 2024-11-24T04:50:55,029 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6cfc9b423cc349b8baad0609572356bb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:55,029 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_ into 6067f4a5f0c0f6bc1ff68dd504605759/a 2024-11-24T04:50:55,029 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile0 into 6067f4a5f0c0f6bc1ff68dd504605759/a (new location: hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_) 2024-11-24T04:50:55,030 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile1 into 6067f4a5f0c0f6bc1ff68dd504605759/a as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_ - updating store file list. 2024-11-24T04:50:55,035 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:55,035 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_ into 6067f4a5f0c0f6bc1ff68dd504605759/a 2024-11-24T04:50:55,035 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile1 into 6067f4a5f0c0f6bc1ff68dd504605759/a (new location: hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_) 2024-11-24T04:50:55,037 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile2 into 6067f4a5f0c0f6bc1ff68dd504605759/a as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_ - updating store file list. 2024-11-24T04:50:55,043 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:55,043 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_ into 6067f4a5f0c0f6bc1ff68dd504605759/a 2024-11-24T04:50:55,043 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44683/hbase/testCompactedBulkLoadedFiles/hfile2 into 6067f4a5f0c0f6bc1ff68dd504605759/a (new location: hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_) 2024-11-24T04:50:55,052 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T04:50:55,052 DEBUG [Time-limited test {}] regionserver.HStore(1541): 6067f4a5f0c0f6bc1ff68dd504605759/a is initiating major compaction (all files) 2024-11-24T04:50:55,053 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 6067f4a5f0c0f6bc1ff68dd504605759/a in testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:55,053 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_] into tmpdir=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/.tmp, totalSize=19.3 K 2024-11-24T04:50:55,053 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ce1ea7583d784049958ed51adb5c41bb, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732423854930 2024-11-24T04:50:55,054 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6cfc9b423cc349b8baad0609572356bb_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:50:55,054 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:50:55,055 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T04:50:55,062 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T04:50:55,062 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:55,063 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T04:50:55,063 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T04:50:55,069 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/.tmp/a/dee0765b7fec47be97e3edd28a09ae64 is 55, key is testCompactedBulkLoadedFiles/a:a/1732423854930/Put/seqid=0 2024-11-24T04:50:55,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:50:55,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:50:55,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741922_1100 (size=6154) 2024-11-24T04:50:55,085 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/.tmp/a/dee0765b7fec47be97e3edd28a09ae64 as hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/dee0765b7fec47be97e3edd28a09ae64 2024-11-24T04:50:55,093 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 6067f4a5f0c0f6bc1ff68dd504605759/a of 6067f4a5f0c0f6bc1ff68dd504605759 into dee0765b7fec47be97e3edd28a09ae64(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T04:50:55,093 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 6067f4a5f0c0f6bc1ff68dd504605759: 2024-11-24T04:50:55,093 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T04:50:55,093 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T04:50:55,129 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896, size=0 (0bytes) 2024-11-24T04:50:55,129 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 might be still open, length is 0 2024-11-24T04:50:55,129 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 2024-11-24T04:50:55,130 WARN [IPC Server handler 1 on default port 44683 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-24T04:50:55,130 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 after 1ms 2024-11-24T04:50:56,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51588 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:46055:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51588 dst: /127.0.0.1:46055 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46055 remote=/127.0.0.1:51588]. Total timeout mills is 60000, 58217 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:56,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:56624 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:42305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56624 dst: /127.0.0.1:42305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:56,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:53918 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:43517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53918 dst: /127.0.0.1:43517 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:56,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741917_1101 (size=1368) 2024-11-24T04:50:56,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741917_1101 (size=1368) 2024-11-24T04:50:56,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741917_1101 (size=1368) 2024-11-24T04:50:59,131 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 after 4001ms 2024-11-24T04:50:59,133 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:59,133 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 took 4004ms 2024-11-24T04:50:59,135 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896; continuing. 2024-11-24T04:50:59,135 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 so closing down 2024-11-24T04:50:59,135 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:59,136 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423854896.temp 2024-11-24T04:50:59,138 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000003-wal.1732423854896.temp 2024-11-24T04:50:59,138 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:59,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741923_1102 (size=635) 2024-11-24T04:50:59,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741923_1102 (size=635) 2024-11-24T04:50:59,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741923_1102 (size=635) 2024-11-24T04:50:59,144 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000003-wal.1732423854896.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:59,146 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000003-wal.1732423854896.temp to hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000008 2024-11-24T04:50:59,146 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:50:59,146 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896, journal: Splitting hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896, size=0 (0bytes) at 1732423855129Finishing writing output for hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 so closing down at 1732423859135 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000003-wal.1732423854896.temp at 1732423859138 (+3 ms)3 split writer threads finished at 1732423859138Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000003-wal.1732423854896.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732423859144 (+6 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000003-wal.1732423854896.temp to hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000008 at 1732423859146 (+2 ms)Processed 5 edits across 1 Regions in 12 ms; skipped=3; WAL=hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896, size=0, length=0, corrupted=false, cancelled=false at 1732423859146 2024-11-24T04:50:59,148 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423854896 2024-11-24T04:50:59,149 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000008 2024-11-24T04:50:59,149 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:59,151 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:59,166 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423859152, exclude list is [], retry=0 2024-11-24T04:50:59,168 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:59,168 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:59,169 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:59,170 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423859152 2024-11-24T04:50:59,170 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:59,170 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6067f4a5f0c0f6bc1ff68dd504605759, NAME => 'testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:59,170 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:59,171 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,171 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,172 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,172 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName a 2024-11-24T04:50:59,173 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,179 DEBUG [StoreFileOpener-6067f4a5f0c0f6bc1ff68dd504605759-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:59,179 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_ 2024-11-24T04:50:59,183 DEBUG [StoreFileOpener-6067f4a5f0c0f6bc1ff68dd504605759-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:59,183 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_ 2024-11-24T04:50:59,186 DEBUG [StoreFileOpener-6067f4a5f0c0f6bc1ff68dd504605759-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 6cfc9b423cc349b8baad0609572356bb_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T04:50:59,186 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_ 2024-11-24T04:50:59,190 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb 2024-11-24T04:50:59,193 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/dee0765b7fec47be97e3edd28a09ae64 2024-11-24T04:50:59,194 WARN [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3b90b03f 2024-11-24T04:50:59,194 WARN [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3b90b03f 2024-11-24T04:50:59,194 WARN [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3b90b03f 2024-11-24T04:50:59,194 WARN [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@3b90b03f 2024-11-24T04:50:59,194 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb] to archive 2024-11-24T04:50:59,195 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T04:50:59,197 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_ to hdfs://localhost:44683/hbase/archive/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_ 2024-11-24T04:50:59,198 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_ to hdfs://localhost:44683/hbase/archive/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_ 2024-11-24T04:50:59,199 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_ to hdfs://localhost:44683/hbase/archive/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/6cfc9b423cc349b8baad0609572356bb_SeqId_4_ 2024-11-24T04:50:59,200 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb to hdfs://localhost:44683/hbase/archive/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/a/ce1ea7583d784049958ed51adb5c41bb 2024-11-24T04:50:59,201 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,201 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,201 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName b 2024-11-24T04:50:59,202 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,202 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,202 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,203 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6067f4a5f0c0f6bc1ff68dd504605759 columnFamilyName c 2024-11-24T04:50:59,203 DEBUG [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,203 INFO [StoreOpener-6067f4a5f0c0f6bc1ff68dd504605759-1 {}] regionserver.HStore(327): Store=6067f4a5f0c0f6bc1ff68dd504605759/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,203 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,204 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,205 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,206 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000008 2024-11-24T04:50:59,207 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:59,209 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 6067f4a5f0c0f6bc1ff68dd504605759 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "6067f4a5f0c0f6bc1ff68dd504605759" family_name: "a" compaction_input: "ce1ea7583d784049958ed51adb5c41bb" compaction_input: "6cfc9b423cc349b8baad0609572356bb_SeqId_4_" compaction_input: "5dba58aa31ae4aa8aeb52c1d83f7a423_SeqId_4_" compaction_input: "4befc6c8d7384bb18d284e9b9cfa8a1b_SeqId_4_" compaction_output: "dee0765b7fec47be97e3edd28a09ae64" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T04:50:59,209 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T04:50:59,209 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000008 2024-11-24T04:50:59,209 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/0000000000000000008 2024-11-24T04:50:59,211 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,211 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,211 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:59,212 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6067f4a5f0c0f6bc1ff68dd504605759 2024-11-24T04:50:59,215 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testCompactedBulkLoadedFiles/6067f4a5f0c0f6bc1ff68dd504605759/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T04:50:59,216 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6067f4a5f0c0f6bc1ff68dd504605759; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72036676, jitterRate=0.0734301209449768}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:59,216 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6067f4a5f0c0f6bc1ff68dd504605759: Writing region info on filesystem at 1732423859171Initializing all the Stores at 1732423859171Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859171Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859172 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859172Cleaning up temporary data from old regions at 1732423859211 (+39 ms)Region opened successfully at 1732423859216 (+5 ms) 2024-11-24T04:50:59,218 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6067f4a5f0c0f6bc1ff68dd504605759, disabling compactions & flushes 2024-11-24T04:50:59,218 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:59,218 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:59,218 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. after waiting 0 ms 2024-11-24T04:50:59,218 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:59,219 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732423854839.6067f4a5f0c0f6bc1ff68dd504605759. 2024-11-24T04:50:59,219 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6067f4a5f0c0f6bc1ff68dd504605759: Waiting for close lock at 1732423859218Disabling compacts and flushes for region at 1732423859218Disabling writes for close at 1732423859218Writing region close event to WAL at 1732423859219 (+1 ms)Closed at 1732423859219 2024-11-24T04:50:59,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741924_1103 (size=93) 2024-11-24T04:50:59,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741924_1103 (size=93) 2024-11-24T04:50:59,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741924_1103 (size=93) 2024-11-24T04:50:59,223 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:59,223 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423859152) 2024-11-24T04:50:59,235 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testCompactedBulkLoadedFiles Thread=443 (was 431) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44683 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44683 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_771420193_22 at /127.0.0.1:51666 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_771420193_22 at /127.0.0.1:53984 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=374 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=10438 (was 10613) 2024-11-24T04:50:59,235 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T04:50:59,247 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=374, ProcessCount=11, AvailableMemoryMB=10437 2024-11-24T04:50:59,247 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-11-24T04:50:59,261 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:59,262 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T04:50:59,263 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T04:50:59,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-96986677, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/hregion-96986677, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:59,276 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-96986677/hregion-96986677.1732423859265, exclude list is [], retry=0 2024-11-24T04:50:59,279 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:59,280 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:59,280 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:59,282 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-96986677/hregion-96986677.1732423859265 2024-11-24T04:50:59,282 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:50:59,282 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f4f3a810a4a0b54643729e206c280211, NAME => 'testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44683/hbase 2024-11-24T04:50:59,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:50:59,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:50:59,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741926_1105 (size=67) 2024-11-24T04:50:59,290 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:59,291 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,292 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName a 2024-11-24T04:50:59,293 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,293 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,293 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,294 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName b 2024-11-24T04:50:59,294 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,295 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,295 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,296 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName c 2024-11-24T04:50:59,296 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,297 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,297 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,298 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,298 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,299 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,299 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,299 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:59,300 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,302 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T04:50:59,303 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f4f3a810a4a0b54643729e206c280211; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62475173, jitterRate=-0.06904737651348114}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:59,303 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f4f3a810a4a0b54643729e206c280211: Writing region info on filesystem at 1732423859290Initializing all the Stores at 1732423859291 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859291Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859291Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859291Cleaning up temporary data from old regions at 1732423859299 (+8 ms)Region opened successfully at 1732423859303 (+4 ms) 2024-11-24T04:50:59,303 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f4f3a810a4a0b54643729e206c280211, disabling compactions & flushes 2024-11-24T04:50:59,303 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,303 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,303 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. after waiting 0 ms 2024-11-24T04:50:59,303 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,304 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,304 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f4f3a810a4a0b54643729e206c280211: Waiting for close lock at 1732423859303Disabling compacts and flushes for region at 1732423859303Disabling writes for close at 1732423859303Writing region close event to WAL at 1732423859304 (+1 ms)Closed at 1732423859304 2024-11-24T04:50:59,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741925_1104 (size=93) 2024-11-24T04:50:59,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741925_1104 (size=93) 2024-11-24T04:50:59,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741925_1104 (size=93) 2024-11-24T04:50:59,312 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:50:59,312 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-96986677:(num 1732423859265) 2024-11-24T04:50:59,312 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:59,314 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:59,326 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314, exclude list is [], retry=0 2024-11-24T04:50:59,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:59,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:59,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:59,331 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314 2024-11-24T04:50:59,331 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:59,331 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f4f3a810a4a0b54643729e206c280211, NAME => 'testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:59,331 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:59,332 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,332 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,333 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,334 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName a 2024-11-24T04:50:59,334 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,334 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,334 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,335 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName b 2024-11-24T04:50:59,335 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,335 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,335 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,336 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName c 2024-11-24T04:50:59,336 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,336 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,337 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,337 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,338 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,339 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,339 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,339 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:59,341 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,342 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f4f3a810a4a0b54643729e206c280211; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60157983, jitterRate=-0.10357619822025299}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:59,342 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f4f3a810a4a0b54643729e206c280211: Writing region info on filesystem at 1732423859332Initializing all the Stores at 1732423859332Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859333 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859333Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859333Cleaning up temporary data from old regions at 1732423859339 (+6 ms)Region opened successfully at 1732423859342 (+3 ms) 2024-11-24T04:50:59,349 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f4f3a810a4a0b54643729e206c280211 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T04:50:59,364 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/a/b61f52e376da496889f594d3017f28e9 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732423859342/Put/seqid=0 2024-11-24T04:50:59,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:50:59,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:50:59,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741928_1107 (size=5958) 2024-11-24T04:50:59,371 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/a/b61f52e376da496889f594d3017f28e9 2024-11-24T04:50:59,376 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/a/b61f52e376da496889f594d3017f28e9 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/a/b61f52e376da496889f594d3017f28e9 2024-11-24T04:50:59,381 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/a/b61f52e376da496889f594d3017f28e9, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T04:50:59,382 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f4f3a810a4a0b54643729e206c280211 in 33ms, sequenceid=13, compaction requested=false 2024-11-24T04:50:59,382 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f4f3a810a4a0b54643729e206c280211: 2024-11-24T04:50:59,400 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f4f3a810a4a0b54643729e206c280211, disabling compactions & flushes 2024-11-24T04:50:59,400 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,400 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,400 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. after waiting 0 ms 2024-11-24T04:50:59,400 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,401 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,401 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:50:59,401 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f4f3a810a4a0b54643729e206c280211: Waiting for close lock at 1732423859400Disabling compacts and flushes for region at 1732423859400Disabling writes for close at 1732423859400Writing region close event to WAL at 1732423859401 (+1 ms)Closed at 1732423859401 2024-11-24T04:50:59,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741927_1106 (size=2805) 2024-11-24T04:50:59,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741927_1106 (size=2805) 2024-11-24T04:50:59,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741927_1106 (size=2805) 2024-11-24T04:50:59,418 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314, size=2.7 K (2805bytes) 2024-11-24T04:50:59,418 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314 2024-11-24T04:50:59,419 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314 after 0ms 2024-11-24T04:50:59,420 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:59,421 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314 took 3ms 2024-11-24T04:50:59,421 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T04:50:59,423 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314 so closing down 2024-11-24T04:50:59,423 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:50:59,424 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732423859314.temp 2024-11-24T04:50:59,425 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000003-wal.1732423859314.temp 2024-11-24T04:50:59,426 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:50:59,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741929_1108 (size=2312) 2024-11-24T04:50:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741929_1108 (size=2312) 2024-11-24T04:50:59,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741929_1108 (size=2312) 2024-11-24T04:50:59,433 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000003-wal.1732423859314.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T04:50:59,434 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000003-wal.1732423859314.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000035 2024-11-24T04:50:59,435 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314, size=2.7 K, length=2805, corrupted=false, cancelled=false 2024-11-24T04:50:59,435 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314, journal: Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314, size=2.7 K (2805bytes) at 1732423859418Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314 so closing down at 1732423859423 (+5 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000003-wal.1732423859314.temp at 1732423859425 (+2 ms)3 split writer threads finished at 1732423859426 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000003-wal.1732423859314.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732423859433 (+7 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000003-wal.1732423859314.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000035 at 1732423859434 (+1 ms)Processed 32 edits across 1 Regions in 13 ms; skipped=2; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314, size=2.7 K, length=2805, corrupted=false, cancelled=false at 1732423859435 (+1 ms) 2024-11-24T04:50:59,436 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859314 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423859314 2024-11-24T04:50:59,437 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000035 2024-11-24T04:50:59,437 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:50:59,438 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:50:59,451 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439, exclude list is [], retry=0 2024-11-24T04:50:59,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:50:59,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:50:59,454 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:50:59,456 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 2024-11-24T04:50:59,456 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959),(127.0.0.1/127.0.0.1:44567:44567)] 2024-11-24T04:50:59,456 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f4f3a810a4a0b54643729e206c280211, NAME => 'testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211.', STARTKEY => '', ENDKEY => ''} 2024-11-24T04:50:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:50:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,457 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,458 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,459 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName a 2024-11-24T04:50:59,459 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,464 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/a/b61f52e376da496889f594d3017f28e9 2024-11-24T04:50:59,464 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,464 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,465 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName b 2024-11-24T04:50:59,465 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,466 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,466 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,467 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName c 2024-11-24T04:50:59,467 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:50:59,468 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:50:59,468 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,468 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,470 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,470 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000035 2024-11-24T04:50:59,472 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:50:59,473 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000035 2024-11-24T04:50:59,473 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f4f3a810a4a0b54643729e206c280211 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T04:50:59,487 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/b/e190e8355cfb4e4e862c262305cb7fe0 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732423859382/Put/seqid=0 2024-11-24T04:50:59,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:50:59,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:50:59,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741931_1110 (size=5958) 2024-11-24T04:50:59,494 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/b/e190e8355cfb4e4e862c262305cb7fe0 2024-11-24T04:50:59,518 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/c/dbce901eeda74e4081451050103f3a08 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732423859389/Put/seqid=0 2024-11-24T04:50:59,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:50:59,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:50:59,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741932_1111 (size=5958) 2024-11-24T04:50:59,525 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/c/dbce901eeda74e4081451050103f3a08 2024-11-24T04:50:59,530 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/b/e190e8355cfb4e4e862c262305cb7fe0 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/b/e190e8355cfb4e4e862c262305cb7fe0 2024-11-24T04:50:59,535 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/b/e190e8355cfb4e4e862c262305cb7fe0, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T04:50:59,536 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/c/dbce901eeda74e4081451050103f3a08 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/c/dbce901eeda74e4081451050103f3a08 2024-11-24T04:50:59,540 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/c/dbce901eeda74e4081451050103f3a08, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T04:50:59,541 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f4f3a810a4a0b54643729e206c280211 in 68ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T04:50:59,541 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000035 2024-11-24T04:50:59,542 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,542 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,543 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:50:59,544 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:50:59,546 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T04:50:59,547 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f4f3a810a4a0b54643729e206c280211; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70997555, jitterRate=0.057946011424064636}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:50:59,547 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f4f3a810a4a0b54643729e206c280211: Writing region info on filesystem at 1732423859457Initializing all the Stores at 1732423859458 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859458Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859458Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423859458Obtaining lock to block concurrent updates at 1732423859473 (+15 ms)Preparing flush snapshotting stores in f4f3a810a4a0b54643729e206c280211 at 1732423859473Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732423859473Flushing stores of testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. at 1732423859473Flushing f4f3a810a4a0b54643729e206c280211/b: creating writer at 1732423859473Flushing f4f3a810a4a0b54643729e206c280211/b: appending metadata at 1732423859486 (+13 ms)Flushing f4f3a810a4a0b54643729e206c280211/b: closing flushed file at 1732423859486Flushing f4f3a810a4a0b54643729e206c280211/c: creating writer at 1732423859500 (+14 ms)Flushing f4f3a810a4a0b54643729e206c280211/c: appending metadata at 1732423859517 (+17 ms)Flushing f4f3a810a4a0b54643729e206c280211/c: closing flushed file at 1732423859517Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10e4c179: reopening flushed file at 1732423859529 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@405cf197: reopening flushed file at 1732423859535 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for f4f3a810a4a0b54643729e206c280211 in 68ms, sequenceid=35, compaction requested=false; wal=null at 1732423859541 (+6 ms)Cleaning up temporary data from old regions at 1732423859542 (+1 ms)Region opened successfully at 1732423859547 (+5 ms) 2024-11-24T04:50:59,607 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439, size=0 (0bytes) 2024-11-24T04:50:59,607 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 might be still open, length is 0 2024-11-24T04:50:59,607 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 2024-11-24T04:50:59,607 WARN [IPC Server handler 3 on default port 44683 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-24T04:50:59,607 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 after 0ms 2024-11-24T04:50:59,875 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:54032 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:43517:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54032 dst: /127.0.0.1:43517 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43517 remote=/127.0.0.1:54032]. Total timeout mills is 60000, 59697 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:59,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:56744 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:42305:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56744 dst: /127.0.0.1:42305 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:59,876 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1408394644_22 at /127.0.0.1:51718 [Receiving block BP-1774581998-172.17.0.2-1732423819869:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:46055:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:51718 dst: /127.0.0.1:46055 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:50:59,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741930_1112 (size=2304) 2024-11-24T04:50:59,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741930_1112 (size=2304) 2024-11-24T04:51:03,608 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 after 4001ms 2024-11-24T04:51:03,611 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:51:03,611 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 took 4005ms 2024-11-24T04:51:03,613 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439; continuing. 2024-11-24T04:51:03,613 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 so closing down 2024-11-24T04:51:03,613 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T04:51:03,614 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732423859439.temp 2024-11-24T04:51:03,615 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000037-wal.1732423859439.temp 2024-11-24T04:51:03,615 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T04:51:03,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741933_1113 (size=2312) 2024-11-24T04:51:03,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741933_1113 (size=2312) 2024-11-24T04:51:03,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741933_1113 (size=2312) 2024-11-24T04:51:03,622 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000037-wal.1732423859439.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T04:51:03,623 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000037-wal.1732423859439.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000066 2024-11-24T04:51:03,623 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T04:51:03,623 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439, journal: Splitting hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439, size=0 (0bytes) at 1732423859607Finishing writing output for hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 so closing down at 1732423863613 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000037-wal.1732423859439.temp at 1732423863615 (+2 ms)3 split writer threads finished at 1732423863615Closed recovered edits writer path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000037-wal.1732423859439.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732423863622 (+7 ms)Rename recovered edits hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000037-wal.1732423859439.temp to hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000066 at 1732423863623 (+1 ms)Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439, size=0, length=0, corrupted=false, cancelled=false at 1732423863623 2024-11-24T04:51:03,625 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 to hdfs://localhost:44683/hbase/oldWALs/wal.1732423859439 2024-11-24T04:51:03,625 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000066 2024-11-24T04:51:03,626 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T04:51:03,628 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44683/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260, archiveDir=hdfs://localhost:44683/hbase/oldWALs, maxLogs=32 2024-11-24T04:51:03,647 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423863628, exclude list is [], retry=0 2024-11-24T04:51:03,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42305,DS-b03b3b9e-3702-4dbd-af41-70e09bd5d2f7,DISK] 2024-11-24T04:51:03,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43517,DS-1802503b-9d87-44a3-86e5-5385b2a93562,DISK] 2024-11-24T04:51:03,650 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46055,DS-5b7a04c0-5eec-4454-824e-302d129fa12a,DISK] 2024-11-24T04:51:03,652 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423863628 2024-11-24T04:51:03,652 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44567:44567),(127.0.0.1/127.0.0.1:41661:41661),(127.0.0.1/127.0.0.1:38959:38959)] 2024-11-24T04:51:03,652 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T04:51:03,653 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,654 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName a 2024-11-24T04:51:03,654 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:51:03,659 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/a/b61f52e376da496889f594d3017f28e9 2024-11-24T04:51:03,659 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:51:03,660 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,660 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName b 2024-11-24T04:51:03,660 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:51:03,666 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/b/e190e8355cfb4e4e862c262305cb7fe0 2024-11-24T04:51:03,666 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:51:03,667 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,668 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4f3a810a4a0b54643729e206c280211 columnFamilyName c 2024-11-24T04:51:03,668 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T04:51:03,673 DEBUG [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/c/dbce901eeda74e4081451050103f3a08 2024-11-24T04:51:03,673 INFO [StoreOpener-f4f3a810a4a0b54643729e206c280211-1 {}] regionserver.HStore(327): Store=f4f3a810a4a0b54643729e206c280211/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T04:51:03,673 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,674 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,675 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,676 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000066 2024-11-24T04:51:03,677 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=false, valueCompressionType=null 2024-11-24T04:51:03,681 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000066 2024-11-24T04:51:03,681 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f4f3a810a4a0b54643729e206c280211 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T04:51:03,720 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/a/6e9538151d3547a09530fb1bc6721e70 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732423859554/Put/seqid=0 2024-11-24T04:51:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:51:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:51:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741935_1115 (size=5958) 2024-11-24T04:51:03,726 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/a/6e9538151d3547a09530fb1bc6721e70 2024-11-24T04:51:03,744 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/b/46c4ad77aff44cd583f6bb07077de5de is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732423859561/Put/seqid=0 2024-11-24T04:51:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:51:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:51:03,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741936_1116 (size=5958) 2024-11-24T04:51:03,750 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/b/46c4ad77aff44cd583f6bb07077de5de 2024-11-24T04:51:03,767 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/c/bfadedbdd31f48be866c089b6ca40456 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732423859567/Put/seqid=0 2024-11-24T04:51:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:51:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:51:03,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741937_1117 (size=5958) 2024-11-24T04:51:03,774 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/c/bfadedbdd31f48be866c089b6ca40456 2024-11-24T04:51:03,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/a/6e9538151d3547a09530fb1bc6721e70 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/a/6e9538151d3547a09530fb1bc6721e70 2024-11-24T04:51:03,785 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/a/6e9538151d3547a09530fb1bc6721e70, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:51:03,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/b/46c4ad77aff44cd583f6bb07077de5de as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/b/46c4ad77aff44cd583f6bb07077de5de 2024-11-24T04:51:03,791 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/b/46c4ad77aff44cd583f6bb07077de5de, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:51:03,792 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/.tmp/c/bfadedbdd31f48be866c089b6ca40456 as hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/c/bfadedbdd31f48be866c089b6ca40456 2024-11-24T04:51:03,797 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/c/bfadedbdd31f48be866c089b6ca40456, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T04:51:03,797 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f4f3a810a4a0b54643729e206c280211 in 116ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T04:51:03,798 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/0000000000000000066 2024-11-24T04:51:03,799 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,799 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,800 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T04:51:03,801 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f4f3a810a4a0b54643729e206c280211 2024-11-24T04:51:03,803 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/hbase/data/default/testReplayEditsWrittenViaHRegion/f4f3a810a4a0b54643729e206c280211/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T04:51:03,804 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f4f3a810a4a0b54643729e206c280211; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63995974, jitterRate=-0.04638567566871643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T04:51:03,805 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f4f3a810a4a0b54643729e206c280211: Writing region info on filesystem at 1732423863652Initializing all the Stores at 1732423863653 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423863653Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423863653Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732423863653Obtaining lock to block concurrent updates at 1732423863681 (+28 ms)Preparing flush snapshotting stores in f4f3a810a4a0b54643729e206c280211 at 1732423863681Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732423863681Flushing stores of testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. at 1732423863681Flushing f4f3a810a4a0b54643729e206c280211/a: creating writer at 1732423863681Flushing f4f3a810a4a0b54643729e206c280211/a: appending metadata at 1732423863720 (+39 ms)Flushing f4f3a810a4a0b54643729e206c280211/a: closing flushed file at 1732423863720Flushing f4f3a810a4a0b54643729e206c280211/b: creating writer at 1732423863731 (+11 ms)Flushing f4f3a810a4a0b54643729e206c280211/b: appending metadata at 1732423863744 (+13 ms)Flushing f4f3a810a4a0b54643729e206c280211/b: closing flushed file at 1732423863744Flushing f4f3a810a4a0b54643729e206c280211/c: creating writer at 1732423863755 (+11 ms)Flushing f4f3a810a4a0b54643729e206c280211/c: appending metadata at 1732423863767 (+12 ms)Flushing f4f3a810a4a0b54643729e206c280211/c: closing flushed file at 1732423863767Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bd3c516: reopening flushed file at 1732423863779 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69cc16d9: reopening flushed file at 1732423863785 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c531d13: reopening flushed file at 1732423863791 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f4f3a810a4a0b54643729e206c280211 in 116ms, sequenceid=66, compaction requested=false; wal=null at 1732423863797 (+6 ms)Cleaning up temporary data from old regions at 1732423863799 (+2 ms)Region opened successfully at 1732423863804 (+5 ms) 2024-11-24T04:51:03,818 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f4f3a810a4a0b54643729e206c280211, disabling compactions & flushes 2024-11-24T04:51:03,818 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:51:03,818 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:51:03,819 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. after waiting 0 ms 2024-11-24T04:51:03,819 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:51:03,823 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732423859261.f4f3a810a4a0b54643729e206c280211. 2024-11-24T04:51:03,823 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f4f3a810a4a0b54643729e206c280211: Waiting for close lock at 1732423863818Disabling compacts and flushes for region at 1732423863818Disabling writes for close at 1732423863819 (+1 ms)Writing region close event to WAL at 1732423863823 (+4 ms)Closed at 1732423863823 2024-11-24T04:51:03,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741934_1114 (size=93) 2024-11-24T04:51:03,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741934_1114 (size=93) 2024-11-24T04:51:03,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741934_1114 (size=93) 2024-11-24T04:51:03,828 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T04:51:03,828 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732423863628) 2024-11-24T04:51:03,842 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayCompressed#testReplayEditsWrittenViaHRegion Thread=450 (was 443) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009807680_22 at /127.0.0.1:41872 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009807680_22 at /127.0.0.1:56780 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44683 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1987708253) connection to localhost/127.0.0.1:44683 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2009807680_22 at /127.0.0.1:41260 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 374), ProcessCount=11 (was 11), AvailableMemoryMB=10386 (was 10437) 2024-11-24T04:51:03,842 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-24T04:51:03,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T04:51:03,843 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:51:03,843 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:51:03,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:51:03,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:51:03,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T04:51:03,844 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T04:51:03,844 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=938327945, stopped=false 2024-11-24T04:51:03,844 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4464c5b832df,36681,1732423824726 2024-11-24T04:51:03,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:51:03,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:51:03,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T04:51:03,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:51:03,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:51:03,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:51:03,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:51:03,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:51:03,920 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T04:51:03,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:51:03,920 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:51:03,920 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:51:03,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T04:51:03,920 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4464c5b832df,46377,1732423825594' ***** 2024-11-24T04:51:03,920 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T04:51:03,920 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4464c5b832df,42181,1732423825646' ***** 2024-11-24T04:51:03,920 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T04:51:03,921 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:51:03,921 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T04:51:03,921 INFO [RS:1;4464c5b832df:46377 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T04:51:03,921 INFO [RS:2;4464c5b832df:42181 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T04:51:03,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:51:03,921 INFO [RS:1;4464c5b832df:46377 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T04:51:03,921 INFO [RS:2;4464c5b832df:42181 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T04:51:03,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T04:51:03,921 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(3091): Received CLOSE for 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:51:03,921 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(959): stopping server 4464c5b832df,46377,1732423825594 2024-11-24T04:51:03,921 INFO [RS:1;4464c5b832df:46377 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:51:03,921 INFO [RS:1;4464c5b832df:46377 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4464c5b832df:46377. 2024-11-24T04:51:03,921 DEBUG [RS:1;4464c5b832df:46377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:51:03,921 DEBUG [RS:1;4464c5b832df:46377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:51:03,921 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(976): stopping server 4464c5b832df,46377,1732423825594; all regions closed. 2024-11-24T04:51:03,922 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(959): stopping server 4464c5b832df,42181,1732423825646 2024-11-24T04:51:03,922 INFO [RS:2;4464c5b832df:42181 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:51:03,922 INFO [RS:2;4464c5b832df:42181 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;4464c5b832df:42181. 2024-11-24T04:51:03,922 DEBUG [RS:2;4464c5b832df:42181 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T04:51:03,922 DEBUG [RS:2;4464c5b832df:42181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:51:03,922 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8a87c32a42ac0dc0089a1412f6f2216d, disabling compactions & flushes 2024-11-24T04:51:03,922 INFO [RS:2;4464c5b832df:42181 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:51:03,922 INFO [RS:2;4464c5b832df:42181 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:51:03,922 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:51:03,922 INFO [RS:2;4464c5b832df:42181 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:51:03,922 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:51:03,922 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. after waiting 0 ms 2024-11-24T04:51:03,922 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T04:51:03,922 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:51:03,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741833_1009 (size=2054) 2024-11-24T04:51:03,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741833_1009 (size=2054) 2024-11-24T04:51:03,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741833_1009 (size=2054) 2024-11-24T04:51:03,927 DEBUG [RS:1;4464c5b832df:46377 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4464c5b832df%2C46377%2C1732423825594:(num 1732423827167) 2024-11-24T04:51:03,927 DEBUG [RS:1;4464c5b832df:46377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:51:03,927 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T04:51:03,927 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 8a87c32a42ac0dc0089a1412f6f2216d=testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d.} 2024-11-24T04:51:03,927 DEBUG [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8a87c32a42ac0dc0089a1412f6f2216d 2024-11-24T04:51:03,927 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T04:51:03,927 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] hbase.ChoreService(370): Chore service for: regionserver/4464c5b832df:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:51:03,927 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T04:51:03,927 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T04:51:03,927 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T04:51:03,927 INFO [regionserver/4464c5b832df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T04:51:03,927 INFO [RS:1;4464c5b832df:46377 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:51:03,927 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-24T04:51:03,928 INFO [RS:1;4464c5b832df:46377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46377 2024-11-24T04:51:03,932 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/default/testReplayEditsAfterRegionMovedWithMultiCF/8a87c32a42ac0dc0089a1412f6f2216d/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T04:51:03,933 INFO [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:51:03,933 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8a87c32a42ac0dc0089a1412f6f2216d: Waiting for close lock at 1732423863922Running coprocessor pre-close hooks at 1732423863922Disabling compacts and flushes for region at 1732423863922Disabling writes for close at 1732423863922Writing region close event to WAL at 1732423863929 (+7 ms)Running coprocessor post-close hooks at 1732423863933 (+4 ms)Closed at 1732423863933 2024-11-24T04:51:03,933 DEBUG [RS_CLOSE_REGION-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d. 2024-11-24T04:51:03,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4464c5b832df,46377,1732423825594 2024-11-24T04:51:03,939 INFO [RS:1;4464c5b832df:46377 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:51:03,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:51:03,949 INFO [regionserver/4464c5b832df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:51:03,950 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4464c5b832df,46377,1732423825594] 2024-11-24T04:51:03,951 INFO [regionserver/4464c5b832df:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:51:03,954 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/info/4061e34311fd48f7a10914844cd18737 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732423840647.8a87c32a42ac0dc0089a1412f6f2216d./info:regioninfo/1732423843809/Put/seqid=0 2024-11-24T04:51:03,960 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4464c5b832df,46377,1732423825594 already deleted, retry=false 2024-11-24T04:51:03,961 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4464c5b832df,46377,1732423825594 expired; onlineServers=1 2024-11-24T04:51:03,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:51:03,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:51:03,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741938_1118 (size=8243) 2024-11-24T04:51:03,965 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/info/4061e34311fd48f7a10914844cd18737 2024-11-24T04:51:03,985 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/ns/11f600f65d55479997d723c7c784ea9a is 43, key is default/ns:d/1732423827829/Put/seqid=0 2024-11-24T04:51:03,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:51:03,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:51:03,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741939_1119 (size=5153) 2024-11-24T04:51:03,992 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/ns/11f600f65d55479997d723c7c784ea9a 2024-11-24T04:51:04,016 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/table/921ca52b61184696bbd49866cfb5a1e1 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732423841083/Put/seqid=0 2024-11-24T04:51:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:51:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:51:04,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741940_1120 (size=5431) 2024-11-24T04:51:04,022 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/table/921ca52b61184696bbd49866cfb5a1e1 2024-11-24T04:51:04,028 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/info/4061e34311fd48f7a10914844cd18737 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/info/4061e34311fd48f7a10914844cd18737 2024-11-24T04:51:04,034 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/info/4061e34311fd48f7a10914844cd18737, entries=18, sequenceid=21, filesize=8.0 K 2024-11-24T04:51:04,035 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/ns/11f600f65d55479997d723c7c784ea9a as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/ns/11f600f65d55479997d723c7c784ea9a 2024-11-24T04:51:04,042 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/ns/11f600f65d55479997d723c7c784ea9a, entries=2, sequenceid=21, filesize=5.0 K 2024-11-24T04:51:04,043 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/.tmp/table/921ca52b61184696bbd49866cfb5a1e1 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/table/921ca52b61184696bbd49866cfb5a1e1 2024-11-24T04:51:04,048 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/table/921ca52b61184696bbd49866cfb5a1e1, entries=2, sequenceid=21, filesize=5.3 K 2024-11-24T04:51:04,049 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=21, compaction requested=false 2024-11-24T04:51:04,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:51:04,050 INFO [RS:1;4464c5b832df:46377 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:51:04,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46377-0x1016b2dadec0002, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:51:04,050 INFO [RS:1;4464c5b832df:46377 {}] regionserver.HRegionServer(1031): Exiting; stopping=4464c5b832df,46377,1732423825594; zookeeper connection closed. 2024-11-24T04:51:04,050 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7b63ad35 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7b63ad35 2024-11-24T04:51:04,054 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-24T04:51:04,054 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T04:51:04,054 INFO [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T04:51:04,054 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732423863927Running coprocessor pre-close hooks at 1732423863927Disabling compacts and flushes for region at 1732423863927Disabling writes for close at 1732423863927Obtaining lock to block concurrent updates at 1732423863927Preparing flush snapshotting stores in 1588230740 at 1732423863928 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732423863929 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732423863929Flushing 1588230740/info: creating writer at 1732423863929Flushing 1588230740/info: appending metadata at 1732423863953 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732423863954 (+1 ms)Flushing 1588230740/ns: creating writer at 1732423863971 (+17 ms)Flushing 1588230740/ns: appending metadata at 1732423863985 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732423863985Flushing 1588230740/table: creating writer at 1732423863997 (+12 ms)Flushing 1588230740/table: appending metadata at 1732423864016 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732423864016Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c80c87a: reopening flushed file at 1732423864027 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6be3a4a5: reopening flushed file at 1732423864034 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5907ce47: reopening flushed file at 1732423864042 (+8 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 122ms, sequenceid=21, compaction requested=false at 1732423864049 (+7 ms)Writing region close event to WAL at 1732423864050 (+1 ms)Running coprocessor post-close hooks at 1732423864054 (+4 ms)Closed at 1732423864054 2024-11-24T04:51:04,055 DEBUG [RS_CLOSE_META-regionserver/4464c5b832df:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T04:51:04,066 INFO [regionserver/4464c5b832df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-24T04:51:04,066 INFO [regionserver/4464c5b832df:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-24T04:51:04,127 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(976): stopping server 4464c5b832df,42181,1732423825646; all regions closed. 2024-11-24T04:51:04,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741836_1012 (size=3783) 2024-11-24T04:51:04,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741836_1012 (size=3783) 2024-11-24T04:51:04,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741836_1012 (size=3783) 2024-11-24T04:51:04,132 DEBUG [RS:2;4464c5b832df:42181 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs 2024-11-24T04:51:04,132 INFO [RS:2;4464c5b832df:42181 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4464c5b832df%2C42181%2C1732423825646.meta:.meta(num 1732423827604) 2024-11-24T04:51:04,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741834_1010 (size=841) 2024-11-24T04:51:04,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741834_1010 (size=841) 2024-11-24T04:51:04,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741834_1010 (size=841) 2024-11-24T04:51:04,136 DEBUG [RS:2;4464c5b832df:42181 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/oldWALs 2024-11-24T04:51:04,136 INFO [RS:2;4464c5b832df:42181 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4464c5b832df%2C42181%2C1732423825646:(num 1732423827167) 2024-11-24T04:51:04,136 DEBUG [RS:2;4464c5b832df:42181 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T04:51:04,136 INFO [RS:2;4464c5b832df:42181 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T04:51:04,136 INFO [RS:2;4464c5b832df:42181 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:51:04,136 INFO [RS:2;4464c5b832df:42181 {}] hbase.ChoreService(370): Chore service for: regionserver/4464c5b832df:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T04:51:04,136 INFO [RS:2;4464c5b832df:42181 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:51:04,136 INFO [regionserver/4464c5b832df:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:51:04,136 INFO [RS:2;4464c5b832df:42181 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42181 2024-11-24T04:51:04,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4464c5b832df,42181,1732423825646 2024-11-24T04:51:04,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T04:51:04,146 INFO [RS:2;4464c5b832df:42181 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:51:04,146 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4464c5b832df,42181,1732423825646] 2024-11-24T04:51:04,167 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4464c5b832df,42181,1732423825646 already deleted, retry=false 2024-11-24T04:51:04,167 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4464c5b832df,42181,1732423825646 expired; onlineServers=0 2024-11-24T04:51:04,167 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4464c5b832df,36681,1732423824726' ***** 2024-11-24T04:51:04,167 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T04:51:04,167 INFO [M:0;4464c5b832df:36681 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T04:51:04,167 INFO [M:0;4464c5b832df:36681 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T04:51:04,167 DEBUG [M:0;4464c5b832df:36681 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T04:51:04,167 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T04:51:04,167 DEBUG [M:0;4464c5b832df:36681 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T04:51:04,167 DEBUG [master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.large.0-1732423826758 {}] cleaner.HFileCleaner(306): Exit Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.large.0-1732423826758,5,FailOnTimeoutGroup] 2024-11-24T04:51:04,168 DEBUG [master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.small.0-1732423826759 {}] cleaner.HFileCleaner(306): Exit Thread[master/4464c5b832df:0:becomeActiveMaster-HFileCleaner.small.0-1732423826759,5,FailOnTimeoutGroup] 2024-11-24T04:51:04,168 INFO [M:0;4464c5b832df:36681 {}] hbase.ChoreService(370): Chore service for: master/4464c5b832df:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T04:51:04,168 INFO [M:0;4464c5b832df:36681 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T04:51:04,168 DEBUG [M:0;4464c5b832df:36681 {}] master.HMaster(1795): Stopping service threads 2024-11-24T04:51:04,168 INFO [M:0;4464c5b832df:36681 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T04:51:04,168 INFO [M:0;4464c5b832df:36681 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T04:51:04,169 INFO [M:0;4464c5b832df:36681 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T04:51:04,169 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T04:51:04,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T04:51:04,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T04:51:04,219 DEBUG [M:0;4464c5b832df:36681 {}] zookeeper.ZKUtil(347): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T04:51:04,220 WARN [M:0;4464c5b832df:36681 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T04:51:04,220 INFO [M:0;4464c5b832df:36681 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/.lastflushedseqids 2024-11-24T04:51:04,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:51:04,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:51:04,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741941_1121 (size=138) 2024-11-24T04:51:04,232 INFO [M:0;4464c5b832df:36681 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T04:51:04,233 INFO [M:0;4464c5b832df:36681 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T04:51:04,233 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T04:51:04,233 INFO [M:0;4464c5b832df:36681 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:51:04,233 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:51:04,233 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T04:51:04,233 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:51:04,233 INFO [M:0;4464c5b832df:36681 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-11-24T04:51:04,246 DEBUG [M:0;4464c5b832df:36681 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a56d41883d3407da45699b284dd6bb5 is 82, key is hbase:meta,,1/info:regioninfo/1732423827685/Put/seqid=0 2024-11-24T04:51:04,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:51:04,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:51:04,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741942_1122 (size=5672) 2024-11-24T04:51:04,253 INFO [M:0;4464c5b832df:36681 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a56d41883d3407da45699b284dd6bb5 2024-11-24T04:51:04,256 INFO [RS:2;4464c5b832df:42181 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:51:04,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:51:04,256 INFO [RS:2;4464c5b832df:42181 {}] regionserver.HRegionServer(1031): Exiting; stopping=4464c5b832df,42181,1732423825646; zookeeper connection closed. 2024-11-24T04:51:04,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42181-0x1016b2dadec0003, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:51:04,257 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e94a572 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e94a572 2024-11-24T04:51:04,257 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T04:51:04,278 DEBUG [M:0;4464c5b832df:36681 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/42bf556a885a4de8a7c569e9573ae078 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732423841093/Put/seqid=0 2024-11-24T04:51:04,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741943_1123 (size=7756) 2024-11-24T04:51:04,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741943_1123 (size=7756) 2024-11-24T04:51:04,293 INFO [M:0;4464c5b832df:36681 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/42bf556a885a4de8a7c569e9573ae078 2024-11-24T04:51:04,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741943_1123 (size=7756) 2024-11-24T04:51:04,298 INFO [M:0;4464c5b832df:36681 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 42bf556a885a4de8a7c569e9573ae078 2024-11-24T04:51:04,318 DEBUG [M:0;4464c5b832df:36681 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9b4eb9014a0747dab2af86e5d0a1bb91 is 69, key is 4464c5b832df,42181,1732423825646/rs:state/1732423826822/Put/seqid=0 2024-11-24T04:51:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741944_1124 (size=5440) 2024-11-24T04:51:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741944_1124 (size=5440) 2024-11-24T04:51:04,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741944_1124 (size=5440) 2024-11-24T04:51:04,327 INFO [M:0;4464c5b832df:36681 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9b4eb9014a0747dab2af86e5d0a1bb91 2024-11-24T04:51:04,331 INFO [M:0;4464c5b832df:36681 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b4eb9014a0747dab2af86e5d0a1bb91 2024-11-24T04:51:04,332 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7a56d41883d3407da45699b284dd6bb5 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7a56d41883d3407da45699b284dd6bb5 2024-11-24T04:51:04,337 INFO [M:0;4464c5b832df:36681 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7a56d41883d3407da45699b284dd6bb5, entries=8, sequenceid=168, filesize=5.5 K 2024-11-24T04:51:04,338 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/42bf556a885a4de8a7c569e9573ae078 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/42bf556a885a4de8a7c569e9573ae078 2024-11-24T04:51:04,342 INFO [M:0;4464c5b832df:36681 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 42bf556a885a4de8a7c569e9573ae078 2024-11-24T04:51:04,343 INFO [M:0;4464c5b832df:36681 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/42bf556a885a4de8a7c569e9573ae078, entries=17, sequenceid=168, filesize=7.6 K 2024-11-24T04:51:04,344 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9b4eb9014a0747dab2af86e5d0a1bb91 as hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9b4eb9014a0747dab2af86e5d0a1bb91 2024-11-24T04:51:04,348 INFO [M:0;4464c5b832df:36681 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b4eb9014a0747dab2af86e5d0a1bb91 2024-11-24T04:51:04,348 INFO [M:0;4464c5b832df:36681 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44683/user/jenkins/test-data/1ce5204e-c90b-c459-f30e-fa26cd8f3322/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9b4eb9014a0747dab2af86e5d0a1bb91, entries=3, sequenceid=168, filesize=5.3 K 2024-11-24T04:51:04,350 INFO [M:0;4464c5b832df:36681 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=168, compaction requested=false 2024-11-24T04:51:04,351 INFO [M:0;4464c5b832df:36681 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T04:51:04,351 DEBUG [M:0;4464c5b832df:36681 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732423864233Disabling compacts and flushes for region at 1732423864233Disabling writes for close at 1732423864233Obtaining lock to block concurrent updates at 1732423864233Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732423864233Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1732423864233Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732423864234 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732423864234Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732423864246 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732423864246Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732423864257 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732423864277 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732423864277Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732423864298 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732423864317 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732423864317Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26c617fa: reopening flushed file at 1732423864331 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c657a3a: reopening flushed file at 1732423864337 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2af4aa02: reopening flushed file at 1732423864343 (+6 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=168, compaction requested=false at 1732423864350 (+7 ms)Writing region close event to WAL at 1732423864351 (+1 ms)Closed at 1732423864351 2024-11-24T04:51:04,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43517 is added to blk_1073741830_1006 (size=71349) 2024-11-24T04:51:04,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42305 is added to blk_1073741830_1006 (size=71349) 2024-11-24T04:51:04,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46055 is added to blk_1073741830_1006 (size=71349) 2024-11-24T04:51:04,354 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T04:51:04,354 INFO [M:0;4464c5b832df:36681 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T04:51:04,354 INFO [M:0;4464c5b832df:36681 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36681 2024-11-24T04:51:04,354 INFO [M:0;4464c5b832df:36681 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T04:51:04,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:51:04,482 INFO [M:0;4464c5b832df:36681 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T04:51:04,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36681-0x1016b2dadec0000, quorum=127.0.0.1:49287, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T04:51:04,489 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844456 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844456 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732423844120/wal.1732423844456 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:51:04,490 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732423836165/wal.1732423836340 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:51:04,491 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732423859260/wal.1732423859439 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:51:04,493 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423853935 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423853935 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732423844603/wal.1732423853935 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:51:04,494 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732423828932/wal.1732423829406 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:51:04,496 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854691 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854691 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732423854162/wal.1732423854691 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:51:04,497 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732423854838/wal.1732423854896 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T04:51:04,500 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal.1732423828805 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal.1732423828805 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732423828596/wal.1732423828805 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:51:04,503 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal.1732423828338 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal.1732423828338 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732423828099/wal.1732423828338 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1408394644_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-24T04:51:04,509 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4290616c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:51:04,511 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5e25d2b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:51:04,511 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:51:04,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6853f5bc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:51:04,512 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bf0fdb4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,STOPPED} 2024-11-24T04:51:04,515 WARN [BP-1774581998-172.17.0.2-1732423819869 heartbeating to localhost/127.0.0.1:44683 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:51:04,515 WARN [BP-1774581998-172.17.0.2-1732423819869 heartbeating to localhost/127.0.0.1:44683 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1774581998-172.17.0.2-1732423819869 (Datanode Uuid 024efb5a-2ac3-457e-9045-db84e38ed4a0) service to localhost/127.0.0.1:44683 2024-11-24T04:51:04,517 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data6/current/BP-1774581998-172.17.0.2-1732423819869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:51:04,517 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:51:04,517 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:51:04,517 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:51:04,519 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28e1ba78{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:51:04,520 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@702c0733{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:51:04,520 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data5/current/BP-1774581998-172.17.0.2-1732423819869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:51:04,520 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:51:04,520 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3264f7d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:51:04,521 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e971547{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,STOPPED} 2024-11-24T04:51:04,522 WARN [BP-1774581998-172.17.0.2-1732423819869 heartbeating to localhost/127.0.0.1:44683 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:51:04,522 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:51:04,522 WARN [BP-1774581998-172.17.0.2-1732423819869 heartbeating to localhost/127.0.0.1:44683 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1774581998-172.17.0.2-1732423819869 (Datanode Uuid f93db99e-cdc0-4f99-a3b0-7770d06b2821) service to localhost/127.0.0.1:44683 2024-11-24T04:51:04,522 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:51:04,523 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data3/current/BP-1774581998-172.17.0.2-1732423819869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:51:04,523 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:51:04,524 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data4/current/BP-1774581998-172.17.0.2-1732423819869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:51:04,529 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4dc44ec6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T04:51:04,530 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@17f6e478{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:51:04,530 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:51:04,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d33512e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:51:04,530 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4b895d95{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,STOPPED} 2024-11-24T04:51:04,532 WARN [BP-1774581998-172.17.0.2-1732423819869 heartbeating to localhost/127.0.0.1:44683 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T04:51:04,532 WARN [BP-1774581998-172.17.0.2-1732423819869 heartbeating to localhost/127.0.0.1:44683 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1774581998-172.17.0.2-1732423819869 (Datanode Uuid 19d587c1-17c9-409b-ac27-73e2c9f40c5d) service to localhost/127.0.0.1:44683 2024-11-24T04:51:04,532 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T04:51:04,532 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T04:51:04,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data1/current/BP-1774581998-172.17.0.2-1732423819869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:51:04,533 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/cluster_75614da4-eb81-fddf-bdeb-f3cf44778217/data/data2/current/BP-1774581998-172.17.0.2-1732423819869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T04:51:04,536 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T04:51:04,544 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1843526c{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T04:51:04,545 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@776c4d96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T04:51:04,546 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T04:51:04,546 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70596a48{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T04:51:04,546 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6630be9c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ce56fba7-837b-6d56-7943-1ba57e39f933/hadoop.log.dir/,STOPPED} 2024-11-24T04:51:04,563 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T04:51:04,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down