2024-11-25 03:48:09,757 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-25 03:48:09,773 main DEBUG Took 0.013737 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 03:48:09,774 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 03:48:09,774 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 03:48:09,776 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 03:48:09,778 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,787 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 03:48:09,808 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,810 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,811 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,813 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,814 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,817 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,818 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,818 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,819 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,819 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,820 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,820 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,821 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,822 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,822 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,823 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 03:48:09,824 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,824 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 03:48:09,826 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 03:48:09,828 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 03:48:09,829 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 03:48:09,831 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 03:48:09,833 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 03:48:09,833 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 03:48:09,844 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 03:48:09,847 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 03:48:09,849 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 03:48:09,850 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 03:48:09,851 main DEBUG createAppenders(={Console}) 2024-11-25 03:48:09,852 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-25 03:48:09,852 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-25 03:48:09,853 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-25 03:48:09,854 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 03:48:09,854 main DEBUG OutputStream closed 2024-11-25 03:48:09,854 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 03:48:09,855 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 03:48:09,856 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-25 03:48:09,963 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 03:48:09,965 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 03:48:09,967 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 03:48:09,968 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 03:48:09,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 03:48:09,969 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 03:48:09,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 03:48:09,970 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 03:48:09,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 03:48:09,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 03:48:09,971 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 03:48:09,972 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 03:48:09,973 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 03:48:09,974 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 03:48:09,977 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 03:48:09,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 03:48:09,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 03:48:09,980 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 03:48:09,983 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 03:48:09,984 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-25 03:48:09,984 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 03:48:09,985 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-25T03:48:10,393 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0 2024-11-25 03:48:10,397 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 03:48:10,397 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T03:48:10,408 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-25T03:48:10,416 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-25T03:48:10,447 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T03:48:10,499 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T03:48:10,500 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T03:48:10,516 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-25T03:48:10,534 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a, deleteOnExit=true 2024-11-25T03:48:10,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-25T03:48:10,536 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/test.cache.data in system properties and HBase conf 2024-11-25T03:48:10,537 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.tmp.dir in system properties and HBase conf 2024-11-25T03:48:10,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir in system properties and HBase conf 2024-11-25T03:48:10,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-25T03:48:10,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-25T03:48:10,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-25T03:48:10,649 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T03:48:10,822 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T03:48:10,830 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:48:10,831 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-25T03:48:10,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-25T03:48:10,832 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:48:10,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-25T03:48:10,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-25T03:48:10,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-25T03:48:10,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:48:10,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-25T03:48:10,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/nfs.dump.dir in system properties and HBase conf 2024-11-25T03:48:10,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/java.io.tmpdir in system properties and HBase conf 2024-11-25T03:48:10,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-25T03:48:10,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-25T03:48:10,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-25T03:48:12,184 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-25T03:48:12,275 INFO [Time-limited test {}] log.Log(170): Logging initialized @3499ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-25T03:48:12,348 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:48:12,414 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:48:12,440 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:48:12,441 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:48:12,442 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:48:12,468 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:48:12,472 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:48:12,473 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:48:12,699 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5608f426{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/java.io.tmpdir/jetty-localhost-33609-hadoop-hdfs-3_4_1-tests_jar-_-any-2251214684561608847/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:48:12,705 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:33609} 2024-11-25T03:48:12,705 INFO [Time-limited test {}] server.Server(415): Started @3929ms 2024-11-25T03:48:13,345 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:48:13,354 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:48:13,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:48:13,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:48:13,355 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:48:13,356 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5cf706e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:48:13,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e9da5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:48:13,471 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@45b57af8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/java.io.tmpdir/jetty-localhost-41107-hadoop-hdfs-3_4_1-tests_jar-_-any-5238363010314281777/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:48:13,472 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@461b5ea8{HTTP/1.1, (http/1.1)}{localhost:41107} 2024-11-25T03:48:13,472 INFO [Time-limited test {}] server.Server(415): Started @4696ms 2024-11-25T03:48:13,523 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:48:13,683 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:48:13,710 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:48:13,740 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:48:13,740 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:48:13,741 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-25T03:48:13,749 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b034666{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:48:13,751 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40a3a086{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:48:13,897 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@44df5d9d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/java.io.tmpdir/jetty-localhost-41165-hadoop-hdfs-3_4_1-tests_jar-_-any-8792678527343000882/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:48:13,899 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@56fbb88e{HTTP/1.1, (http/1.1)}{localhost:41165} 2024-11-25T03:48:13,899 INFO [Time-limited test {}] server.Server(415): Started @5123ms 2024-11-25T03:48:13,902 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:48:13,955 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-25T03:48:13,961 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-25T03:48:13,967 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-25T03:48:13,967 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-25T03:48:13,968 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-25T03:48:13,969 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7161498b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,AVAILABLE} 2024-11-25T03:48:13,969 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1846121a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-25T03:48:14,090 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6f0cddd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/java.io.tmpdir/jetty-localhost-37313-hadoop-hdfs-3_4_1-tests_jar-_-any-13062043556545180867/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:48:14,091 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@25cdce3{HTTP/1.1, (http/1.1)}{localhost:37313} 2024-11-25T03:48:14,091 INFO [Time-limited test {}] server.Server(415): Started @5315ms 2024-11-25T03:48:14,093 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-25T03:48:15,865 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data2/current/BP-1013946533-172.17.0.2-1732506491574/current, will proceed with Du for space computation calculation, 2024-11-25T03:48:15,865 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data3/current/BP-1013946533-172.17.0.2-1732506491574/current, will proceed with Du for space computation calculation, 2024-11-25T03:48:15,865 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data1/current/BP-1013946533-172.17.0.2-1732506491574/current, will proceed with Du for space computation calculation, 2024-11-25T03:48:15,866 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data4/current/BP-1013946533-172.17.0.2-1732506491574/current, will proceed with Du for space computation calculation, 2024-11-25T03:48:15,917 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:48:15,917 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:48:15,933 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data5/current/BP-1013946533-172.17.0.2-1732506491574/current, will proceed with Du for space computation calculation, 2024-11-25T03:48:15,935 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data6/current/BP-1013946533-172.17.0.2-1732506491574/current, will proceed with Du for space computation calculation, 2024-11-25T03:48:15,986 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-25T03:48:15,993 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfdbbb362513cb0a4 with lease ID 0xd287c9e648b091b7: Processing first storage report for DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74 from datanode DatanodeRegistration(127.0.0.1:34347, datanodeUuid=d70f36d4-ff73-46a6-b535-9c9b5b1f19f5, infoPort=40539, infoSecurePort=0, ipcPort=46055, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574) 2024-11-25T03:48:15,995 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfdbbb362513cb0a4 with lease ID 0xd287c9e648b091b7: from storage DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74 node DatanodeRegistration(127.0.0.1:34347, datanodeUuid=d70f36d4-ff73-46a6-b535-9c9b5b1f19f5, infoPort=40539, infoSecurePort=0, ipcPort=46055, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T03:48:15,995 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdd0ef10c7d24e50e with lease ID 0xd287c9e648b091b6: Processing first storage report for DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae from datanode DatanodeRegistration(127.0.0.1:33729, datanodeUuid=4315fb89-1f23-4d4f-8c92-1866ea4ff287, infoPort=32831, infoSecurePort=0, ipcPort=42859, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574) 2024-11-25T03:48:15,995 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd0ef10c7d24e50e with lease ID 0xd287c9e648b091b6: from storage DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae node DatanodeRegistration(127.0.0.1:33729, datanodeUuid=4315fb89-1f23-4d4f-8c92-1866ea4ff287, infoPort=32831, infoSecurePort=0, ipcPort=42859, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:48:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f173f48d565df9a with lease ID 0xd287c9e648b091b8: Processing first storage report for DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969 from datanode DatanodeRegistration(127.0.0.1:43937, datanodeUuid=4c6f3230-8958-4157-8900-e7142a6de5d9, infoPort=39943, infoSecurePort=0, ipcPort=34415, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574) 2024-11-25T03:48:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f173f48d565df9a with lease ID 0xd287c9e648b091b8: from storage DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969 node DatanodeRegistration(127.0.0.1:43937, datanodeUuid=4c6f3230-8958-4157-8900-e7142a6de5d9, infoPort=39943, infoSecurePort=0, ipcPort=34415, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-25T03:48:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfdbbb362513cb0a4 with lease ID 0xd287c9e648b091b7: Processing first storage report for DS-97f64422-6fdd-4c3e-9dcc-ab98e72fd484 from datanode DatanodeRegistration(127.0.0.1:34347, datanodeUuid=d70f36d4-ff73-46a6-b535-9c9b5b1f19f5, infoPort=40539, infoSecurePort=0, ipcPort=46055, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574) 2024-11-25T03:48:15,996 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfdbbb362513cb0a4 with lease ID 0xd287c9e648b091b7: from storage DS-97f64422-6fdd-4c3e-9dcc-ab98e72fd484 node DatanodeRegistration(127.0.0.1:34347, datanodeUuid=d70f36d4-ff73-46a6-b535-9c9b5b1f19f5, infoPort=40539, infoSecurePort=0, ipcPort=46055, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:48:15,997 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdd0ef10c7d24e50e with lease ID 0xd287c9e648b091b6: Processing first storage report for DS-17fd6519-16d7-4f23-b711-6bebb81c2772 from datanode DatanodeRegistration(127.0.0.1:33729, datanodeUuid=4315fb89-1f23-4d4f-8c92-1866ea4ff287, infoPort=32831, infoSecurePort=0, ipcPort=42859, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574) 2024-11-25T03:48:15,997 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdd0ef10c7d24e50e with lease ID 0xd287c9e648b091b6: from storage DS-17fd6519-16d7-4f23-b711-6bebb81c2772 node DatanodeRegistration(127.0.0.1:33729, datanodeUuid=4315fb89-1f23-4d4f-8c92-1866ea4ff287, infoPort=32831, infoSecurePort=0, ipcPort=42859, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:48:15,997 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5f173f48d565df9a with lease ID 0xd287c9e648b091b8: Processing first storage report for DS-5d8c18db-b535-4133-8b48-e02ffdafdf78 from datanode DatanodeRegistration(127.0.0.1:43937, datanodeUuid=4c6f3230-8958-4157-8900-e7142a6de5d9, infoPort=39943, infoSecurePort=0, ipcPort=34415, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574) 2024-11-25T03:48:15,997 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5f173f48d565df9a with lease ID 0xd287c9e648b091b8: from storage DS-5d8c18db-b535-4133-8b48-e02ffdafdf78 node DatanodeRegistration(127.0.0.1:43937, datanodeUuid=4c6f3230-8958-4157-8900-e7142a6de5d9, infoPort=39943, infoSecurePort=0, ipcPort=34415, storageInfo=lv=-57;cid=testClusterID;nsid=1453207211;c=1732506491574), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-25T03:48:16,029 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0 2024-11-25T03:48:16,120 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/zookeeper_0, clientPort=57852, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T03:48:16,131 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57852 2024-11-25T03:48:16,164 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:16,167 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:16,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:48:16,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:48:16,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741825_1001 (size=7) 2024-11-25T03:48:16,811 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2 with version=8 2024-11-25T03:48:16,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/hbase-staging 2024-11-25T03:48:17,093 INFO [Time-limited test {}] client.ConnectionUtils(128): master/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:48:17,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,108 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:48:17,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,108 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:48:17,236 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T03:48:17,296 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T03:48:17,305 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T03:48:17,309 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:48:17,335 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 49338 (auto-detected) 2024-11-25T03:48:17,336 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-25T03:48:17,355 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41803 2024-11-25T03:48:17,376 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41803 connecting to ZooKeeper ensemble=127.0.0.1:57852 2024-11-25T03:48:17,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:418030x0, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:48:17,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41803-0x101701b280d0000 connected 2024-11-25T03:48:17,603 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,606 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,617 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:48:17,621 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2, hbase.cluster.distributed=false 2024-11-25T03:48:17,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:48:17,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41803 2024-11-25T03:48:17,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41803 2024-11-25T03:48:17,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41803 2024-11-25T03:48:17,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41803 2024-11-25T03:48:17,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41803 2024-11-25T03:48:17,742 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:48:17,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,743 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,743 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:48:17,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:48:17,746 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:48:17,749 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:48:17,749 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33171 2024-11-25T03:48:17,751 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33171 connecting to ZooKeeper ensemble=127.0.0.1:57852 2024-11-25T03:48:17,752 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,755 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:331710x0, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:48:17,770 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:48:17,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33171-0x101701b280d0001 connected 2024-11-25T03:48:17,776 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:48:17,784 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:48:17,786 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:48:17,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:48:17,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33171 2024-11-25T03:48:17,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33171 2024-11-25T03:48:17,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33171 2024-11-25T03:48:17,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33171 2024-11-25T03:48:17,793 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33171 2024-11-25T03:48:17,810 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:48:17,810 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,810 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,810 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:48:17,810 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,811 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:48:17,811 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:48:17,811 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:48:17,812 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33219 2024-11-25T03:48:17,814 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33219 connecting to ZooKeeper ensemble=127.0.0.1:57852 2024-11-25T03:48:17,816 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,818 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:332190x0, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:48:17,866 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:332190x0, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:48:17,866 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33219-0x101701b280d0002 connected 2024-11-25T03:48:17,867 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:48:17,868 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:48:17,870 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:48:17,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:48:17,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33219 2024-11-25T03:48:17,876 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33219 2024-11-25T03:48:17,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33219 2024-11-25T03:48:17,880 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33219 2024-11-25T03:48:17,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33219 2024-11-25T03:48:17,897 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/d73269e94d72:0 server-side Connection retries=45 2024-11-25T03:48:17,898 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,898 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,898 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T03:48:17,898 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T03:48:17,899 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T03:48:17,899 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T03:48:17,899 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T03:48:17,900 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38451 2024-11-25T03:48:17,902 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38451 connecting to ZooKeeper ensemble=127.0.0.1:57852 2024-11-25T03:48:17,903 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,906 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:17,917 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384510x0, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T03:48:17,918 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:384510x0, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:48:17,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38451-0x101701b280d0003 connected 2024-11-25T03:48:17,918 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T03:48:17,919 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T03:48:17,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T03:48:17,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T03:48:17,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38451 2024-11-25T03:48:17,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38451 2024-11-25T03:48:17,923 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38451 2024-11-25T03:48:17,924 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38451 2024-11-25T03:48:17,925 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38451 2024-11-25T03:48:17,939 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;d73269e94d72:41803 2024-11-25T03:48:17,940 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/d73269e94d72,41803,1732506496940 2024-11-25T03:48:17,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:17,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:17,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:17,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:17,958 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/d73269e94d72,41803,1732506496940 2024-11-25T03:48:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:48:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:48:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T03:48:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:17,989 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T03:48:17,990 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/d73269e94d72,41803,1732506496940 from backup master directory 2024-11-25T03:48:18,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/d73269e94d72,41803,1732506496940 2024-11-25T03:48:18,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:18,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:18,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:18,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T03:48:18,002 WARN [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:48:18,002 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=d73269e94d72,41803,1732506496940 2024-11-25T03:48:18,005 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T03:48:18,006 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T03:48:18,059 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/hbase.id] with ID: 26f49762-00a5-45ef-9268-18b170fa2906 2024-11-25T03:48:18,060 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/.tmp/hbase.id 2024-11-25T03:48:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:48:18,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:48:18,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741826_1002 (size=42) 2024-11-25T03:48:18,077 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/.tmp/hbase.id]:[hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/hbase.id] 2024-11-25T03:48:18,120 INFO [master/d73269e94d72:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-25T03:48:18,125 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T03:48:18,142 INFO [master/d73269e94d72:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-11-25T03:48:18,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,169 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:48:18,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:48:18,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741827_1003 (size=196) 2024-11-25T03:48:18,207 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:48:18,209 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T03:48:18,214 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:48:18,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:48:18,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741828_1004 (size=1189) 2024-11-25T03:48:18,259 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store 2024-11-25T03:48:18,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:48:18,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:48:18,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741829_1005 (size=34) 2024-11-25T03:48:18,285 INFO [master/d73269e94d72:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T03:48:18,290 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:18,292 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:48:18,292 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:48:18,292 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:48:18,295 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-25T03:48:18,295 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:48:18,295 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:48:18,297 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506498292Disabling compacts and flushes for region at 1732506498292Disabling writes for close at 1732506498295 (+3 ms)Writing region close event to WAL at 1732506498295Closed at 1732506498295 2024-11-25T03:48:18,299 WARN [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/.initializing 2024-11-25T03:48:18,299 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/WALs/d73269e94d72,41803,1732506496940 2024-11-25T03:48:18,307 INFO [master/d73269e94d72:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:18,324 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C41803%2C1732506496940, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/WALs/d73269e94d72,41803,1732506496940, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/oldWALs, maxLogs=10 2024-11-25T03:48:18,350 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/WALs/d73269e94d72,41803,1732506496940/d73269e94d72%2C41803%2C1732506496940.1732506498328, exclude list is [], retry=0 2024-11-25T03:48:18,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:18,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:18,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:18,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:18,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-25T03:48:18,402 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-25T03:48:18,459 INFO [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/WALs/d73269e94d72,41803,1732506496940/d73269e94d72%2C41803%2C1732506496940.1732506498328 2024-11-25T03:48:18,459 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:18,460 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:18,460 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:18,463 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,463 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T03:48:18,519 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:18,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:18,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T03:48:18,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:18,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:18,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T03:48:18,529 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:18,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:18,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T03:48:18,533 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:18,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:18,535 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,538 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,539 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,544 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,545 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,548 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:48:18,550 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T03:48:18,554 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:18,555 INFO [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73471741, jitterRate=0.0948142558336258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:48:18,562 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732506498474Initializing all the Stores at 1732506498476 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506498476Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506498477 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506498477Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506498477Cleaning up temporary data from old regions at 1732506498545 (+68 ms)Region opened successfully at 1732506498562 (+17 ms) 2024-11-25T03:48:18,563 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T03:48:18,593 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b6172cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:48:18,620 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T03:48:18,629 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T03:48:18,629 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T03:48:18,632 INFO [master/d73269e94d72:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T03:48:18,633 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-25T03:48:18,637 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-25T03:48:18,637 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T03:48:18,660 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T03:48:18,668 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T03:48:18,717 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T03:48:18,720 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T03:48:18,723 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T03:48:18,734 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T03:48:18,737 INFO [master/d73269e94d72:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T03:48:18,741 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T03:48:18,748 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T03:48:18,750 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T03:48:18,758 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T03:48:18,781 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T03:48:18,790 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,805 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=d73269e94d72,41803,1732506496940, sessionid=0x101701b280d0000, setting cluster-up flag (Was=false) 2024-11-25T03:48:18,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,864 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T03:48:18,867 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,41803,1732506496940 2024-11-25T03:48:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:18,927 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T03:48:18,931 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=d73269e94d72,41803,1732506496940 2024-11-25T03:48:18,940 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T03:48:19,014 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T03:48:19,022 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T03:48:19,029 INFO [master/d73269e94d72:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T03:48:19,029 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(746): ClusterId : 26f49762-00a5-45ef-9268-18b170fa2906 2024-11-25T03:48:19,029 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(746): ClusterId : 26f49762-00a5-45ef-9268-18b170fa2906 2024-11-25T03:48:19,029 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(746): ClusterId : 26f49762-00a5-45ef-9268-18b170fa2906 2024-11-25T03:48:19,031 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:48:19,031 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:48:19,031 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T03:48:19,036 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: d73269e94d72,41803,1732506496940 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T03:48:19,055 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:48:19,055 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:48:19,055 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T03:48:19,055 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:48:19,055 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:48:19,055 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T03:48:19,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:48:19,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:48:19,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:48:19,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/d73269e94d72:0, corePoolSize=5, maxPoolSize=5 2024-11-25T03:48:19,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/d73269e94d72:0, corePoolSize=10, maxPoolSize=10 2024-11-25T03:48:19,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,056 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:48:19,057 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,058 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732506529058 2024-11-25T03:48:19,059 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T03:48:19,060 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T03:48:19,062 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:48:19,063 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T03:48:19,064 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T03:48:19,065 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T03:48:19,065 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T03:48:19,065 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T03:48:19,065 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:48:19,065 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:48:19,066 DEBUG [RS:1;d73269e94d72:33219 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ffff8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:48:19,066 DEBUG [RS:0;d73269e94d72:33171 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2963a1bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:48:19,067 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T03:48:19,067 DEBUG [RS:2;d73269e94d72:38451 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f186959, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=d73269e94d72/172.17.0.2:0 2024-11-25T03:48:19,066 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,070 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:19,071 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T03:48:19,072 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T03:48:19,073 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T03:48:19,074 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T03:48:19,076 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T03:48:19,076 INFO [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T03:48:19,082 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506499078,5,FailOnTimeoutGroup] 2024-11-25T03:48:19,082 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506499082,5,FailOnTimeoutGroup] 2024-11-25T03:48:19,083 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,083 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T03:48:19,083 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;d73269e94d72:33171 2024-11-25T03:48:19,084 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;d73269e94d72:33219 2024-11-25T03:48:19,084 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,084 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,087 INFO [RS:1;d73269e94d72:33219 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:48:19,087 INFO [RS:0;d73269e94d72:33171 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:48:19,087 INFO [RS:1;d73269e94d72:33219 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:48:19,087 INFO [RS:0;d73269e94d72:33171 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:48:19,087 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:48:19,087 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:48:19,088 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;d73269e94d72:38451 2024-11-25T03:48:19,088 INFO [RS:2;d73269e94d72:38451 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T03:48:19,089 INFO [RS:2;d73269e94d72:38451 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T03:48:19,089 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T03:48:19,090 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,41803,1732506496940 with port=33219, startcode=1732506497809 2024-11-25T03:48:19,090 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,41803,1732506496940 with port=38451, startcode=1732506497897 2024-11-25T03:48:19,090 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(2659): reportForDuty to master=d73269e94d72,41803,1732506496940 with port=33171, startcode=1732506497711 2024-11-25T03:48:19,102 DEBUG [RS:0;d73269e94d72:33171 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:48:19,102 DEBUG [RS:2;d73269e94d72:38451 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:48:19,102 DEBUG [RS:1;d73269e94d72:33219 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T03:48:19,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:48:19,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:48:19,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741831_1007 (size=1321) 2024-11-25T03:48:19,123 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T03:48:19,124 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2 2024-11-25T03:48:19,148 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:48:19,148 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:48:19,148 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T03:48:19,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:48:19,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:48:19,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741832_1008 (size=32) 2024-11-25T03:48:19,155 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,38451,1732506497897 2024-11-25T03:48:19,158 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,38451,1732506497897 2024-11-25T03:48:19,169 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,33171,1732506497711 2024-11-25T03:48:19,170 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,33171,1732506497711 2024-11-25T03:48:19,173 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2 2024-11-25T03:48:19,173 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46839 2024-11-25T03:48:19,173 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:48:19,175 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] master.ServerManager(363): Checking decommissioned status of RegionServer d73269e94d72,33219,1732506497809 2024-11-25T03:48:19,175 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2 2024-11-25T03:48:19,175 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] master.ServerManager(517): Registering regionserver=d73269e94d72,33219,1732506497809 2024-11-25T03:48:19,175 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46839 2024-11-25T03:48:19,175 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:48:19,179 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2 2024-11-25T03:48:19,179 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46839 2024-11-25T03:48:19,179 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T03:48:19,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:48:19,233 DEBUG [RS:0;d73269e94d72:33171 {}] zookeeper.ZKUtil(111): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,33171,1732506497711 2024-11-25T03:48:19,233 DEBUG [RS:2;d73269e94d72:38451 {}] zookeeper.ZKUtil(111): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,38451,1732506497897 2024-11-25T03:48:19,234 WARN [RS:0;d73269e94d72:33171 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:48:19,234 WARN [RS:2;d73269e94d72:38451 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:48:19,234 INFO [RS:2;d73269e94d72:38451 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:19,234 INFO [RS:0;d73269e94d72:33171 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:19,234 DEBUG [RS:1;d73269e94d72:33219 {}] zookeeper.ZKUtil(111): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/d73269e94d72,33219,1732506497809 2024-11-25T03:48:19,234 WARN [RS:1;d73269e94d72:33219 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T03:48:19,235 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33171,1732506497711 2024-11-25T03:48:19,235 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,38451,1732506497897 2024-11-25T03:48:19,235 INFO [RS:1;d73269e94d72:33219 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:19,235 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809 2024-11-25T03:48:19,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,33219,1732506497809] 2024-11-25T03:48:19,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,33171,1732506497711] 2024-11-25T03:48:19,238 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [d73269e94d72,38451,1732506497897] 2024-11-25T03:48:19,265 INFO [RS:1;d73269e94d72:33219 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:48:19,265 INFO [RS:2;d73269e94d72:38451 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:48:19,265 INFO [RS:0;d73269e94d72:33171 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T03:48:19,280 INFO [RS:0;d73269e94d72:33171 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:48:19,280 INFO [RS:1;d73269e94d72:33219 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:48:19,281 INFO [RS:2;d73269e94d72:38451 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T03:48:19,286 INFO [RS:0;d73269e94d72:33171 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:48:19,286 INFO [RS:2;d73269e94d72:38451 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:48:19,286 INFO [RS:1;d73269e94d72:33219 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:48:19,286 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,286 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,286 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,288 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:48:19,288 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:48:19,288 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T03:48:19,295 INFO [RS:0;d73269e94d72:33171 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:48:19,295 INFO [RS:1;d73269e94d72:33219 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:48:19,295 INFO [RS:2;d73269e94d72:38451 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T03:48:19,296 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,296 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,296 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,297 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,297 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:48:19,298 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:48:19,298 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0, corePoolSize=2, maxPoolSize=2 2024-11-25T03:48:19,298 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,298 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,299 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,299 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/d73269e94d72:0, corePoolSize=1, maxPoolSize=1 2024-11-25T03:48:19,299 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:48:19,299 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:48:19,299 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:48:19,299 DEBUG [RS:2;d73269e94d72:38451 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:48:19,299 DEBUG [RS:0;d73269e94d72:33171 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:48:19,299 DEBUG [RS:1;d73269e94d72:33219 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/d73269e94d72:0, corePoolSize=3, maxPoolSize=3 2024-11-25T03:48:19,301 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,301 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,301 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,301 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,301 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,301 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33219,1732506497809-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38451,1732506497897-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:48:19,302 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33171,1732506497711-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:48:19,320 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:48:19,322 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33219,1732506497809-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,323 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,323 INFO [RS:1;d73269e94d72:33219 {}] regionserver.Replication(171): d73269e94d72,33219,1732506497809 started 2024-11-25T03:48:19,325 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:48:19,325 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,33171,1732506497711-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,326 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,326 INFO [RS:0;d73269e94d72:33171 {}] regionserver.Replication(171): d73269e94d72,33171,1732506497711 started 2024-11-25T03:48:19,329 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T03:48:19,329 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,38451,1732506497897-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,329 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,329 INFO [RS:2;d73269e94d72:38451 {}] regionserver.Replication(171): d73269e94d72,38451,1732506497897 started 2024-11-25T03:48:19,345 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,345 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,33219,1732506497809, RpcServer on d73269e94d72/172.17.0.2:33219, sessionid=0x101701b280d0002 2024-11-25T03:48:19,346 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:48:19,346 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,346 DEBUG [RS:1;d73269e94d72:33219 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,33219,1732506497809 2024-11-25T03:48:19,346 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,33171,1732506497711, RpcServer on d73269e94d72/172.17.0.2:33171, sessionid=0x101701b280d0001 2024-11-25T03:48:19,346 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33219,1732506497809' 2024-11-25T03:48:19,347 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:48:19,347 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:48:19,347 DEBUG [RS:0;d73269e94d72:33171 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,33171,1732506497711 2024-11-25T03:48:19,347 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33171,1732506497711' 2024-11-25T03:48:19,347 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:48:19,348 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:48:19,348 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:48:19,348 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:48:19,348 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:48:19,348 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:48:19,348 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:48:19,348 DEBUG [RS:0;d73269e94d72:33171 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,33171,1732506497711 2024-11-25T03:48:19,348 DEBUG [RS:1;d73269e94d72:33219 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,33219,1732506497809 2024-11-25T03:48:19,349 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33171,1732506497711' 2024-11-25T03:48:19,349 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,33219,1732506497809' 2024-11-25T03:48:19,349 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:48:19,349 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:48:19,349 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:48:19,349 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:19,349 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1482): Serving as d73269e94d72,38451,1732506497897, RpcServer on d73269e94d72/172.17.0.2:38451, sessionid=0x101701b280d0003 2024-11-25T03:48:19,349 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:48:19,350 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T03:48:19,350 DEBUG [RS:2;d73269e94d72:38451 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager d73269e94d72,38451,1732506497897 2024-11-25T03:48:19,350 DEBUG [RS:0;d73269e94d72:33171 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:48:19,350 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,38451,1732506497897' 2024-11-25T03:48:19,350 INFO [RS:0;d73269e94d72:33171 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:48:19,350 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T03:48:19,350 INFO [RS:0;d73269e94d72:33171 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:48:19,350 DEBUG [RS:1;d73269e94d72:33219 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:48:19,350 INFO [RS:1;d73269e94d72:33219 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:48:19,350 INFO [RS:1;d73269e94d72:33219 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:48:19,350 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T03:48:19,351 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T03:48:19,351 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T03:48:19,351 DEBUG [RS:2;d73269e94d72:38451 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager d73269e94d72,38451,1732506497897 2024-11-25T03:48:19,351 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'd73269e94d72,38451,1732506497897' 2024-11-25T03:48:19,351 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T03:48:19,352 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T03:48:19,352 DEBUG [RS:2;d73269e94d72:38451 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T03:48:19,352 INFO [RS:2;d73269e94d72:38451 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T03:48:19,352 INFO [RS:2;d73269e94d72:38451 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T03:48:19,458 INFO [RS:2;d73269e94d72:38451 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:19,458 INFO [RS:1;d73269e94d72:33219 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:19,458 INFO [RS:0;d73269e94d72:33171 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:19,462 INFO [RS:2;d73269e94d72:38451 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C38451%2C1732506497897, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,38451,1732506497897, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs, maxLogs=32 2024-11-25T03:48:19,462 INFO [RS:1;d73269e94d72:33219 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C33219%2C1732506497809, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs, maxLogs=32 2024-11-25T03:48:19,462 INFO [RS:0;d73269e94d72:33171 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C33171%2C1732506497711, suffix=, logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33171,1732506497711, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs, maxLogs=32 2024-11-25T03:48:19,480 DEBUG [RS:2;d73269e94d72:38451 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,38451,1732506497897/d73269e94d72%2C38451%2C1732506497897.1732506499467, exclude list is [], retry=0 2024-11-25T03:48:19,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:19,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:19,485 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:19,487 DEBUG [RS:1;d73269e94d72:33219 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809/d73269e94d72%2C33219%2C1732506497809.1732506499467, exclude list is [], retry=0 2024-11-25T03:48:19,487 DEBUG [RS:0;d73269e94d72:33171 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33171,1732506497711/d73269e94d72%2C33171%2C1732506497711.1732506499467, exclude list is [], retry=0 2024-11-25T03:48:19,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:19,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:19,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:19,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:19,497 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:19,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:19,528 INFO [RS:2;d73269e94d72:38451 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,38451,1732506497897/d73269e94d72%2C38451%2C1732506497897.1732506499467 2024-11-25T03:48:19,529 DEBUG [RS:2;d73269e94d72:38451 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:19,531 INFO [RS:0;d73269e94d72:33171 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33171,1732506497711/d73269e94d72%2C33171%2C1732506497711.1732506499467 2024-11-25T03:48:19,532 DEBUG [RS:0;d73269e94d72:33171 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:19,534 INFO [RS:1;d73269e94d72:33219 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809/d73269e94d72%2C33219%2C1732506497809.1732506499467 2024-11-25T03:48:19,534 DEBUG [RS:1;d73269e94d72:33219 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:19,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:19,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:48:19,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:48:19,559 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:19,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:19,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:48:19,563 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:48:19,563 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:19,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:19,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:48:19,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:48:19,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:19,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:19,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:48:19,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:48:19,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:19,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:19,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:48:19,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740 2024-11-25T03:48:19,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740 2024-11-25T03:48:19,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:48:19,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:48:19,577 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:48:19,580 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:48:19,584 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:19,585 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69745976, jitterRate=0.03929603099822998}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:48:19,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732506499554Initializing all the Stores at 1732506499555 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506499555Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506499556 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506499556Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506499556Cleaning up temporary data from old regions at 1732506499576 (+20 ms)Region opened successfully at 1732506499586 (+10 ms) 2024-11-25T03:48:19,586 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:48:19,587 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:48:19,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:48:19,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:48:19,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:48:19,588 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:48:19,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506499586Disabling compacts and flushes for region at 1732506499586Disabling writes for close at 1732506499587 (+1 ms)Writing region close event to WAL at 1732506499588 (+1 ms)Closed at 1732506499588 2024-11-25T03:48:19,592 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:48:19,592 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T03:48:19,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:48:19,608 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:48:19,611 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T03:48:19,764 DEBUG [d73269e94d72:41803 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-25T03:48:19,772 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:48:19,778 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:48:19,778 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:48:19,778 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T03:48:19,779 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:48:19,779 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:48:19,779 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T03:48:19,779 INFO [d73269e94d72:41803 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:48:19,779 INFO [d73269e94d72:41803 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:48:19,779 INFO [d73269e94d72:41803 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T03:48:19,779 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:48:19,786 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,33219,1732506497809 2024-11-25T03:48:19,792 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33219,1732506497809, state=OPENING 2024-11-25T03:48:19,843 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T03:48:19,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:19,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:19,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:19,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:48:19,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:19,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:19,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:19,858 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:19,861 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:48:19,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33219,1732506497809}] 2024-11-25T03:48:20,042 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:48:20,044 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37709, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:48:20,055 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:48:20,055 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:20,056 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:48:20,059 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C33219%2C1732506497809.meta, suffix=.meta, logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs, maxLogs=32 2024-11-25T03:48:20,074 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, exclude list is [], retry=0 2024-11-25T03:48:20,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:20,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:20,078 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:20,109 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta 2024-11-25T03:48:20,109 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:20,109 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:20,111 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:48:20,113 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:48:20,118 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:48:20,122 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:48:20,123 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:20,123 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:48:20,123 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:48:20,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:48:20,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:48:20,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:20,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:20,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:48:20,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:48:20,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:20,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:20,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:48:20,133 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:48:20,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:20,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:20,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:48:20,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:48:20,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:20,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:20,137 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:48:20,138 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740 2024-11-25T03:48:20,141 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740 2024-11-25T03:48:20,143 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:48:20,143 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:48:20,144 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:48:20,147 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:48:20,148 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59009478, jitterRate=-0.120690256357193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:48:20,148 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:48:20,150 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506500123Writing region info on filesystem at 1732506500124 (+1 ms)Initializing all the Stores at 1732506500126 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506500126Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506500126Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506500126Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506500126Cleaning up temporary data from old regions at 1732506500143 (+17 ms)Running coprocessor post-open hooks at 1732506500149 (+6 ms)Region opened successfully at 1732506500149 2024-11-25T03:48:20,157 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732506500033 2024-11-25T03:48:20,168 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:48:20,168 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:48:20,170 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33219,1732506497809 2024-11-25T03:48:20,173 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,33219,1732506497809, state=OPEN 2024-11-25T03:48:20,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:20,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:20,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:20,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:20,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:20,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:20,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:20,222 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:20,223 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,33219,1732506497809 2024-11-25T03:48:20,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T03:48:20,229 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,33219,1732506497809 in 360 msec 2024-11-25T03:48:20,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T03:48:20,237 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 633 msec 2024-11-25T03:48:20,239 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T03:48:20,239 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T03:48:20,256 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:20,258 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1] 2024-11-25T03:48:20,278 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:48:20,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42579, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:48:20,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3310 sec 2024-11-25T03:48:20,303 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732506500303, completionTime=-1 2024-11-25T03:48:20,306 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-25T03:48:20,307 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T03:48:20,348 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-25T03:48:20,348 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732506560348 2024-11-25T03:48:20,348 INFO [master/d73269e94d72:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732506620348 2024-11-25T03:48:20,348 INFO [master/d73269e94d72:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 41 msec 2024-11-25T03:48:20,349 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T03:48:20,356 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,41803,1732506496940-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:20,356 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,41803,1732506496940-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:20,357 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,41803,1732506496940-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:20,358 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-d73269e94d72:41803, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:20,358 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:20,359 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:20,367 DEBUG [master/d73269e94d72:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T03:48:20,388 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.385sec 2024-11-25T03:48:20,389 INFO [master/d73269e94d72:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T03:48:20,390 INFO [master/d73269e94d72:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T03:48:20,391 INFO [master/d73269e94d72:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T03:48:20,391 INFO [master/d73269e94d72:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T03:48:20,391 INFO [master/d73269e94d72:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T03:48:20,392 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,41803,1732506496940-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T03:48:20,393 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,41803,1732506496940-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T03:48:20,397 DEBUG [master/d73269e94d72:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T03:48:20,398 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T03:48:20,399 INFO [master/d73269e94d72:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=d73269e94d72,41803,1732506496940-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:20,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63a09d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:48:20,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request d73269e94d72,41803,-1 for getting cluster id 2024-11-25T03:48:20,447 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T03:48:20,456 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '26f49762-00a5-45ef-9268-18b170fa2906' 2024-11-25T03:48:20,458 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T03:48:20,458 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "26f49762-00a5-45ef-9268-18b170fa2906" 2024-11-25T03:48:20,458 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@659a9655, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:48:20,458 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [d73269e94d72,41803,-1] 2024-11-25T03:48:20,461 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T03:48:20,463 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:48:20,464 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T03:48:20,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328be90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T03:48:20,467 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:20,473 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1] 2024-11-25T03:48:20,473 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:48:20,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45838, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:48:20,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=d73269e94d72,41803,1732506496940 2024-11-25T03:48:20,493 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:46839/hbase 2024-11-25T03:48:20,513 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=364, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=477, ProcessCount=11, AvailableMemoryMB=9041 2024-11-25T03:48:20,536 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:20,539 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:20,540 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:20,544 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92327743, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-92327743, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:20,560 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92327743/hregion-92327743.1732506500546, exclude list is [], retry=0 2024-11-25T03:48:20,564 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:20,565 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:20,566 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:20,609 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92327743/hregion-92327743.1732506500546 2024-11-25T03:48:20,609 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:20,610 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9a71cbf11641912011331c4d08dc3503, NAME => 'testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:20,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741838_1014 (size=64) 2024-11-25T03:48:20,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741838_1014 (size=64) 2024-11-25T03:48:20,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741838_1014 (size=64) 2024-11-25T03:48:20,626 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:20,628 INFO [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:20,631 INFO [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a71cbf11641912011331c4d08dc3503 columnFamilyName a 2024-11-25T03:48:20,631 DEBUG [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:20,632 INFO [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] regionserver.HStore(327): Store=9a71cbf11641912011331c4d08dc3503/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:20,632 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:20,633 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:20,634 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:20,635 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:20,635 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:20,638 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:20,641 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:20,642 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9a71cbf11641912011331c4d08dc3503; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69421686, jitterRate=0.034463733434677124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:20,643 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9a71cbf11641912011331c4d08dc3503: Writing region info on filesystem at 1732506500626Initializing all the Stores at 1732506500628 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506500628Cleaning up temporary data from old regions at 1732506500635 (+7 ms)Region opened successfully at 1732506500643 (+8 ms) 2024-11-25T03:48:20,643 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9a71cbf11641912011331c4d08dc3503, disabling compactions & flushes 2024-11-25T03:48:20,643 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503. 2024-11-25T03:48:20,643 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503. 2024-11-25T03:48:20,643 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503. after waiting 0 ms 2024-11-25T03:48:20,643 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503. 2024-11-25T03:48:20,643 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503. 2024-11-25T03:48:20,644 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9a71cbf11641912011331c4d08dc3503: Waiting for close lock at 1732506500643Disabling compacts and flushes for region at 1732506500643Disabling writes for close at 1732506500643Writing region close event to WAL at 1732506500643Closed at 1732506500643 2024-11-25T03:48:20,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741837_1013 (size=175) 2024-11-25T03:48:20,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741837_1013 (size=175) 2024-11-25T03:48:20,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741837_1013 (size=175) 2024-11-25T03:48:20,655 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:20,655 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92327743:(num 1732506500546) 2024-11-25T03:48:20,657 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T03:48:20,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741839_1015 (size=469) 2024-11-25T03:48:20,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741839_1015 (size=469) 2024-11-25T03:48:20,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741839_1015 (size=469) 2024-11-25T03:48:20,697 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T03:48:20,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741840_1016 (size=322) 2024-11-25T03:48:20,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741840_1016 (size=322) 2024-11-25T03:48:20,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741840_1016 (size=322) 2024-11-25T03:48:20,758 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1, size=469 (469bytes) 2024-11-25T03:48:20,758 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-25T03:48:20,758 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-25T03:48:20,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1 2024-11-25T03:48:20,764 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1 after 4ms 2024-11-25T03:48:20,802 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:20,803 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1 took 47ms 2024-11-25T03:48:20,809 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1 so closing down 2024-11-25T03:48:20,809 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:20,812 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T03:48:20,839 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T03:48:20,840 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:20,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741841_1017 (size=469) 2024-11-25T03:48:20,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741841_1017 (size=469) 2024-11-25T03:48:20,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741841_1017 (size=469) 2024-11-25T03:48:20,855 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:20,858 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002 2024-11-25T03:48:20,863 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 54 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-25T03:48:20,863 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1, size=469 (469bytes) at 1732506500758Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1 so closing down at 1732506500809 (+51 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000001-wal-1.temp at 1732506500839 (+30 ms)3 split writer threads finished at 1732506500840 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732506500855 (+15 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002 at 1732506500858 (+3 ms)Processed 2 edits across 1 Regions in 54 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732506500863 (+5 ms) 2024-11-25T03:48:20,877 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2, size=322 (322bytes) 2024-11-25T03:48:20,877 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2 2024-11-25T03:48:20,878 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2 after 1ms 2024-11-25T03:48:20,908 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:20,908 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2 took 31ms 2024-11-25T03:48:20,911 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2 so closing down 2024-11-25T03:48:20,911 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:20,914 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T03:48:20,941 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T03:48:20,941 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:20,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741842_1018 (size=322) 2024-11-25T03:48:20,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741842_1018 (size=322) 2024-11-25T03:48:20,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741842_1018 (size=322) 2024-11-25T03:48:20,950 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:20,981 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:20,985 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-25T03:48:20,987 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-25T03:48:20,987 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2, size=322 (322bytes) at 1732506500877Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2 so closing down at 1732506500911 (+34 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002-wal-2.temp at 1732506500941 (+30 ms)3 split writer threads finished at 1732506500941Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732506500950 (+9 ms)Processed 1 edits across 1 Regions in 78 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732506500987 (+37 ms) 2024-11-25T03:48:20,987 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:20,989 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:21,003 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal.1732506500990, exclude list is [], retry=0 2024-11-25T03:48:21,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:21,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:21,008 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:21,034 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal.1732506500990 2024-11-25T03:48:21,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:21,035 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9a71cbf11641912011331c4d08dc3503, NAME => 'testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:21,035 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:21,035 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,035 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,038 INFO [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,039 INFO [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9a71cbf11641912011331c4d08dc3503 columnFamilyName a 2024-11-25T03:48:21,039 DEBUG [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:21,040 INFO [StoreOpener-9a71cbf11641912011331c4d08dc3503-1 {}] regionserver.HStore(327): Store=9a71cbf11641912011331c4d08dc3503/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:21,041 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,042 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,045 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,047 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002 2024-11-25T03:48:21,073 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:21,078 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002 2024-11-25T03:48:21,081 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9a71cbf11641912011331c4d08dc3503 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T03:48:21,132 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/.tmp/a/9021a80d266944e0afd5f5e197808449 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732506500656/Put/seqid=0 2024-11-25T03:48:21,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741844_1020 (size=5170) 2024-11-25T03:48:21,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741844_1020 (size=5170) 2024-11-25T03:48:21,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741844_1020 (size=5170) 2024-11-25T03:48:21,144 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/.tmp/a/9021a80d266944e0afd5f5e197808449 2024-11-25T03:48:21,184 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/.tmp/a/9021a80d266944e0afd5f5e197808449 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/a/9021a80d266944e0afd5f5e197808449 2024-11-25T03:48:21,194 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/a/9021a80d266944e0afd5f5e197808449, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T03:48:21,202 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 9a71cbf11641912011331c4d08dc3503 in 117ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T03:48:21,204 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/0000000000000000002 2024-11-25T03:48:21,205 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,205 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,209 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9a71cbf11641912011331c4d08dc3503 2024-11-25T03:48:21,212 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/9a71cbf11641912011331c4d08dc3503/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T03:48:21,214 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9a71cbf11641912011331c4d08dc3503; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60375630, jitterRate=-0.10033300518989563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:21,215 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9a71cbf11641912011331c4d08dc3503: Writing region info on filesystem at 1732506501035Initializing all the Stores at 1732506501037 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506501037Obtaining lock to block concurrent updates at 1732506501081 (+44 ms)Preparing flush snapshotting stores in 9a71cbf11641912011331c4d08dc3503 at 1732506501081Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732506501084 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732506500537.9a71cbf11641912011331c4d08dc3503. at 1732506501084Flushing 9a71cbf11641912011331c4d08dc3503/a: creating writer at 1732506501085 (+1 ms)Flushing 9a71cbf11641912011331c4d08dc3503/a: appending metadata at 1732506501116 (+31 ms)Flushing 9a71cbf11641912011331c4d08dc3503/a: closing flushed file at 1732506501118 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ebdd5e7: reopening flushed file at 1732506501181 (+63 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 9a71cbf11641912011331c4d08dc3503 in 117ms, sequenceid=2, compaction requested=false; wal=null at 1732506501202 (+21 ms)Cleaning up temporary data from old regions at 1732506501205 (+3 ms)Region opened successfully at 1732506501215 (+10 ms) 2024-11-25T03:48:21,247 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=375 (was 364) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:42536 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:48728 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:42382 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:48850 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:41984 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:46839/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:41960 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=477 (was 477), ProcessCount=11 (was 11), AvailableMemoryMB=9025 (was 9041) 2024-11-25T03:48:21,259 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=375, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=477, ProcessCount=11, AvailableMemoryMB=9024 2024-11-25T03:48:21,273 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:21,276 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:21,277 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:21,280 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-94219866, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-94219866, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:21,295 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-94219866/hregion-94219866.1732506501281, exclude list is [], retry=0 2024-11-25T03:48:21,299 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:21,299 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:21,300 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:21,329 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-94219866/hregion-94219866.1732506501281 2024-11-25T03:48:21,329 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:21,330 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dcc83a01e8adc306b32f0f9f3fd15e40, NAME => 'testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:21,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741846_1022 (size=64) 2024-11-25T03:48:21,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741846_1022 (size=64) 2024-11-25T03:48:21,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741846_1022 (size=64) 2024-11-25T03:48:21,342 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:21,344 INFO [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,346 INFO [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc83a01e8adc306b32f0f9f3fd15e40 columnFamilyName a 2024-11-25T03:48:21,346 DEBUG [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:21,347 INFO [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] regionserver.HStore(327): Store=dcc83a01e8adc306b32f0f9f3fd15e40/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:21,347 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,348 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,349 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,349 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,349 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,352 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,355 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:21,356 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc83a01e8adc306b32f0f9f3fd15e40; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61567774, jitterRate=-0.08256867527961731}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:21,357 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc83a01e8adc306b32f0f9f3fd15e40: Writing region info on filesystem at 1732506501342Initializing all the Stores at 1732506501343 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506501343Cleaning up temporary data from old regions at 1732506501349 (+6 ms)Region opened successfully at 1732506501356 (+7 ms) 2024-11-25T03:48:21,357 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dcc83a01e8adc306b32f0f9f3fd15e40, disabling compactions & flushes 2024-11-25T03:48:21,357 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40. 2024-11-25T03:48:21,357 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40. 2024-11-25T03:48:21,357 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40. after waiting 0 ms 2024-11-25T03:48:21,357 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40. 2024-11-25T03:48:21,357 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40. 2024-11-25T03:48:21,358 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dcc83a01e8adc306b32f0f9f3fd15e40: Waiting for close lock at 1732506501357Disabling compacts and flushes for region at 1732506501357Disabling writes for close at 1732506501357Writing region close event to WAL at 1732506501357Closed at 1732506501357 2024-11-25T03:48:21,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741845_1021 (size=175) 2024-11-25T03:48:21,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741845_1021 (size=175) 2024-11-25T03:48:21,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741845_1021 (size=175) 2024-11-25T03:48:21,365 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:21,365 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-94219866:(num 1732506501281) 2024-11-25T03:48:21,366 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T03:48:21,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741847_1023 (size=469) 2024-11-25T03:48:21,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741847_1023 (size=469) 2024-11-25T03:48:21,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741847_1023 (size=469) 2024-11-25T03:48:21,407 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-25T03:48:21,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741848_1024 (size=322) 2024-11-25T03:48:21,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741848_1024 (size=322) 2024-11-25T03:48:21,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741848_1024 (size=322) 2024-11-25T03:48:21,459 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2, size=322 (322bytes) 2024-11-25T03:48:21,459 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2 2024-11-25T03:48:21,460 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2 after 1ms 2024-11-25T03:48:21,488 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:21,488 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2 took 29ms 2024-11-25T03:48:21,491 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2 so closing down 2024-11-25T03:48:21,491 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:21,494 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-25T03:48:21,521 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002-wal-2.temp 2024-11-25T03:48:21,522 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:21,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741849_1025 (size=322) 2024-11-25T03:48:21,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741849_1025 (size=322) 2024-11-25T03:48:21,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741849_1025 (size=322) 2024-11-25T03:48:21,530 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:21,532 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002 2024-11-25T03:48:21,533 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-25T03:48:21,533 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2, size=322 (322bytes) at 1732506501459Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2 so closing down at 1732506501491 (+32 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002-wal-2.temp at 1732506501521 (+30 ms)3 split writer threads finished at 1732506501522 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732506501530 (+8 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002 at 1732506501532 (+2 ms)Processed 1 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732506501533 (+1 ms) 2024-11-25T03:48:21,547 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1, size=469 (469bytes) 2024-11-25T03:48:21,547 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1 2024-11-25T03:48:21,548 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1 after 1ms 2024-11-25T03:48:21,573 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:21,573 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1 took 26ms 2024-11-25T03:48:21,576 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1 so closing down 2024-11-25T03:48:21,576 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:21,578 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-25T03:48:21,603 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000001-wal-1.temp 2024-11-25T03:48:21,604 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:21,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741850_1026 (size=469) 2024-11-25T03:48:21,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741850_1026 (size=469) 2024-11-25T03:48:21,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741850_1026 (size=469) 2024-11-25T03:48:21,611 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:21,645 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:21,648 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002, length=322 2024-11-25T03:48:21,650 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002 2024-11-25T03:48:21,650 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-25T03:48:21,651 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1, journal: Splitting hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1, size=469 (469bytes) at 1732506501547Finishing writing output for hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1 so closing down at 1732506501576 (+29 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000001-wal-1.temp at 1732506501603 (+27 ms)3 split writer threads finished at 1732506501604 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732506501612 (+8 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002 at 1732506501650 (+38 ms)Processed 2 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732506501650 2024-11-25T03:48:21,651 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:21,653 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:21,667 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal.1732506501654, exclude list is [], retry=0 2024-11-25T03:48:21,670 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:21,670 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:21,671 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:21,697 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal.1732506501654 2024-11-25T03:48:21,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dcc83a01e8adc306b32f0f9f3fd15e40, NAME => 'testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,697 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,699 INFO [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,701 INFO [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc83a01e8adc306b32f0f9f3fd15e40 columnFamilyName a 2024-11-25T03:48:21,701 DEBUG [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:21,702 INFO [StoreOpener-dcc83a01e8adc306b32f0f9f3fd15e40-1 {}] regionserver.HStore(327): Store=dcc83a01e8adc306b32f0f9f3fd15e40/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:21,702 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,703 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,705 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,706 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002 2024-11-25T03:48:21,734 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:21,736 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002 2024-11-25T03:48:21,736 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dcc83a01e8adc306b32f0f9f3fd15e40 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-25T03:48:21,752 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/.tmp/a/6ad302655b7b46aab91ac5dcdddd047a is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732506501365/Put/seqid=0 2024-11-25T03:48:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741852_1028 (size=5170) 2024-11-25T03:48:21,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741852_1028 (size=5170) 2024-11-25T03:48:21,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741852_1028 (size=5170) 2024-11-25T03:48:21,763 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/.tmp/a/6ad302655b7b46aab91ac5dcdddd047a 2024-11-25T03:48:21,772 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/.tmp/a/6ad302655b7b46aab91ac5dcdddd047a as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/a/6ad302655b7b46aab91ac5dcdddd047a 2024-11-25T03:48:21,780 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/a/6ad302655b7b46aab91ac5dcdddd047a, entries=2, sequenceid=2, filesize=5.0 K 2024-11-25T03:48:21,781 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for dcc83a01e8adc306b32f0f9f3fd15e40 in 45ms, sequenceid=2, compaction requested=false; wal=null 2024-11-25T03:48:21,782 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/0000000000000000002 2024-11-25T03:48:21,782 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,782 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,784 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc83a01e8adc306b32f0f9f3fd15e40 2024-11-25T03:48:21,786 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/dcc83a01e8adc306b32f0f9f3fd15e40/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-25T03:48:21,787 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc83a01e8adc306b32f0f9f3fd15e40; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67151182, jitterRate=6.305873394012451E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:21,788 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc83a01e8adc306b32f0f9f3fd15e40: Writing region info on filesystem at 1732506501698Initializing all the Stores at 1732506501699 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506501699Obtaining lock to block concurrent updates at 1732506501736 (+37 ms)Preparing flush snapshotting stores in dcc83a01e8adc306b32f0f9f3fd15e40 at 1732506501736Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732506501736Flushing stores of testReplayEditsWrittenIntoWAL,,1732506501274.dcc83a01e8adc306b32f0f9f3fd15e40. at 1732506501736Flushing dcc83a01e8adc306b32f0f9f3fd15e40/a: creating writer at 1732506501736Flushing dcc83a01e8adc306b32f0f9f3fd15e40/a: appending metadata at 1732506501751 (+15 ms)Flushing dcc83a01e8adc306b32f0f9f3fd15e40/a: closing flushed file at 1732506501751Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2550a51a: reopening flushed file at 1732506501771 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for dcc83a01e8adc306b32f0f9f3fd15e40 in 45ms, sequenceid=2, compaction requested=false; wal=null at 1732506501781 (+10 ms)Cleaning up temporary data from old regions at 1732506501782 (+1 ms)Region opened successfully at 1732506501788 (+6 ms) 2024-11-25T03:48:21,802 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 375) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:42604 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:48728 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:42382 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:42044 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:41984 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:48910 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=477 (was 477), ProcessCount=11 (was 11), AvailableMemoryMB=9010 (was 9024) 2024-11-25T03:48:21,814 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=477, ProcessCount=11, AvailableMemoryMB=9009 2024-11-25T03:48:21,834 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:21,836 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:21,837 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:21,841 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70117882, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-70117882, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:21,854 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70117882/hregion-70117882.1732506501841, exclude list is [], retry=0 2024-11-25T03:48:21,857 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:21,857 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:21,858 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:21,884 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70117882/hregion-70117882.1732506501841 2024-11-25T03:48:21,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:21,884 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1db25a25d9d8388dd3bb2b93b9e6aaaf, NAME => 'testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:21,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741854_1030 (size=64) 2024-11-25T03:48:21,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741854_1030 (size=64) 2024-11-25T03:48:21,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741854_1030 (size=64) 2024-11-25T03:48:21,899 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:21,901 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,903 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db25a25d9d8388dd3bb2b93b9e6aaaf columnFamilyName a 2024-11-25T03:48:21,903 DEBUG [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:21,904 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(327): Store=1db25a25d9d8388dd3bb2b93b9e6aaaf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:21,904 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,906 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db25a25d9d8388dd3bb2b93b9e6aaaf columnFamilyName b 2024-11-25T03:48:21,906 DEBUG [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:21,907 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(327): Store=1db25a25d9d8388dd3bb2b93b9e6aaaf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:21,907 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,909 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db25a25d9d8388dd3bb2b93b9e6aaaf columnFamilyName c 2024-11-25T03:48:21,909 DEBUG [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:21,909 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(327): Store=1db25a25d9d8388dd3bb2b93b9e6aaaf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:21,910 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,912 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,912 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,913 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:21,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:21,917 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:21,917 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1db25a25d9d8388dd3bb2b93b9e6aaaf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75432364, jitterRate=0.12402981519699097}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:21,918 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1db25a25d9d8388dd3bb2b93b9e6aaaf: Writing region info on filesystem at 1732506501900Initializing all the Stores at 1732506501901 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506501901Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506501901Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506501901Cleaning up temporary data from old regions at 1732506501912 (+11 ms)Region opened successfully at 1732506501917 (+5 ms) 2024-11-25T03:48:21,918 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1db25a25d9d8388dd3bb2b93b9e6aaaf, disabling compactions & flushes 2024-11-25T03:48:21,918 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:21,918 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:21,918 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. after waiting 0 ms 2024-11-25T03:48:21,918 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:21,918 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:21,918 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1db25a25d9d8388dd3bb2b93b9e6aaaf: Waiting for close lock at 1732506501918Disabling compacts and flushes for region at 1732506501918Disabling writes for close at 1732506501918Writing region close event to WAL at 1732506501918Closed at 1732506501918 2024-11-25T03:48:21,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741853_1029 (size=175) 2024-11-25T03:48:21,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741853_1029 (size=175) 2024-11-25T03:48:21,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741853_1029 (size=175) 2024-11-25T03:48:21,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:21,924 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70117882:(num 1732506501841) 2024-11-25T03:48:21,925 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:21,927 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:21,947 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928, exclude list is [], retry=0 2024-11-25T03:48:21,951 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:21,951 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:21,952 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:21,990 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 2024-11-25T03:48:21,991 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:22,196 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928, size=0 (0bytes) 2024-11-25T03:48:22,196 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 might be still open, length is 0 2024-11-25T03:48:22,196 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 2024-11-25T03:48:22,197 WARN [IPC Server handler 3 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-25T03:48:22,197 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 after 1ms 2024-11-25T03:48:25,010 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:56856 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33729:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56856 dst: /127.0.0.1:33729 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33729 remote=/127.0.0.1:56856]. Total timeout mills is 60000, 57155 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:25,011 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:45652 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45652 dst: /127.0.0.1:34347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:25,011 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:57596 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43937:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57596 dst: /127.0.0.1:43937 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:25,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741855_1032 (size=497775) 2024-11-25T03:48:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741855_1032 (size=497775) 2024-11-25T03:48:25,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741855_1032 (size=497775) 2024-11-25T03:48:25,640 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:48:25,690 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:48:26,198 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 after 4002ms 2024-11-25T03:48:26,232 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:26,233 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 took 4037ms 2024-11-25T03:48:26,238 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732506501928.temp 2024-11-25T03:48:26,276 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000000001-wal.1732506501928.temp 2024-11-25T03:48:26,319 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928; continuing. 2024-11-25T03:48:26,319 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 so closing down 2024-11-25T03:48:26,320 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:26,320 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:26,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741856_1033 (size=497783) 2024-11-25T03:48:26,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741856_1033 (size=497783) 2024-11-25T03:48:26,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741856_1033 (size=497783) 2024-11-25T03:48:26,324 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000000001-wal.1732506501928.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-11-25T03:48:26,326 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000000001-wal.1732506501928.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000003002 2024-11-25T03:48:26,326 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T03:48:26,326 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928, size=0 (0bytes) at 1732506502196Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000000001-wal.1732506501928.temp at 1732506506276 (+4080 ms)Split 1024 edits, skipped 0 edits. at 1732506506281 (+5 ms)Split 2048 edits, skipped 0 edits. at 1732506506301 (+20 ms)Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 so closing down at 1732506506320 (+19 ms)3 split writer threads finished at 1732506506320Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000000001-wal.1732506501928.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1732506506324 (+4 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000000001-wal.1732506501928.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000003002 at 1732506506326 (+2 ms)Processed 3002 edits across 1 Regions in 92 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928, size=0, length=0, corrupted=false, cancelled=false at 1732506506326 2024-11-25T03:48:26,328 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506501928 2024-11-25T03:48:26,330 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000003002 2024-11-25T03:48:26,330 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:26,333 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:26,345 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506506333, exclude list is [], retry=0 2024-11-25T03:48:26,349 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:26,349 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:26,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:26,377 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506506333 2024-11-25T03:48:26,377 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:26,378 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:26,380 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:26,381 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db25a25d9d8388dd3bb2b93b9e6aaaf columnFamilyName a 2024-11-25T03:48:26,381 DEBUG [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:26,382 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(327): Store=1db25a25d9d8388dd3bb2b93b9e6aaaf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:26,382 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:26,384 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db25a25d9d8388dd3bb2b93b9e6aaaf columnFamilyName b 2024-11-25T03:48:26,384 DEBUG [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:26,385 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(327): Store=1db25a25d9d8388dd3bb2b93b9e6aaaf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:26,385 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:26,386 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1db25a25d9d8388dd3bb2b93b9e6aaaf columnFamilyName c 2024-11-25T03:48:26,387 DEBUG [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:26,387 INFO [StoreOpener-1db25a25d9d8388dd3bb2b93b9e6aaaf-1 {}] regionserver.HStore(327): Store=1db25a25d9d8388dd3bb2b93b9e6aaaf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:26,388 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:26,389 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:26,391 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:26,392 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000003002 2024-11-25T03:48:26,418 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:26,451 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T03:48:26,815 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1db25a25d9d8388dd3bb2b93b9e6aaaf 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T03:48:26,852 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/a/e2d090d1edd845ecbb88ca65af63bc06 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732506502000/Put/seqid=0 2024-11-25T03:48:26,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741858_1035 (size=50463) 2024-11-25T03:48:26,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741858_1035 (size=50463) 2024-11-25T03:48:26,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741858_1035 (size=50463) 2024-11-25T03:48:26,862 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/a/e2d090d1edd845ecbb88ca65af63bc06 2024-11-25T03:48:26,871 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/a/e2d090d1edd845ecbb88ca65af63bc06 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/a/e2d090d1edd845ecbb88ca65af63bc06 2024-11-25T03:48:26,879 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/a/e2d090d1edd845ecbb88ca65af63bc06, entries=754, sequenceid=754, filesize=49.3 K 2024-11-25T03:48:26,880 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 1db25a25d9d8388dd3bb2b93b9e6aaaf in 64ms, sequenceid=754, compaction requested=false; wal=null 2024-11-25T03:48:26,899 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T03:48:26,899 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1db25a25d9d8388dd3bb2b93b9e6aaaf 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T03:48:26,908 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/a/3062fab19da141f0bd1183ba26ce52a7 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732506502056/Put/seqid=0 2024-11-25T03:48:26,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741859_1036 (size=20072) 2024-11-25T03:48:26,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741859_1036 (size=20072) 2024-11-25T03:48:26,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741859_1036 (size=20072) 2024-11-25T03:48:26,919 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/a/3062fab19da141f0bd1183ba26ce52a7 2024-11-25T03:48:26,944 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/b/71b1558f68324955a761616e86fbbe7a is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732506502081/Put/seqid=0 2024-11-25T03:48:26,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741860_1037 (size=35835) 2024-11-25T03:48:26,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741860_1037 (size=35835) 2024-11-25T03:48:26,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741860_1037 (size=35835) 2024-11-25T03:48:26,953 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/b/71b1558f68324955a761616e86fbbe7a 2024-11-25T03:48:26,961 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/a/3062fab19da141f0bd1183ba26ce52a7 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/a/3062fab19da141f0bd1183ba26ce52a7 2024-11-25T03:48:26,969 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/a/3062fab19da141f0bd1183ba26ce52a7, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-25T03:48:26,971 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/b/71b1558f68324955a761616e86fbbe7a as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/b/71b1558f68324955a761616e86fbbe7a 2024-11-25T03:48:26,978 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/b/71b1558f68324955a761616e86fbbe7a, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-25T03:48:26,978 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1db25a25d9d8388dd3bb2b93b9e6aaaf in 79ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-25T03:48:26,997 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T03:48:26,998 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1db25a25d9d8388dd3bb2b93b9e6aaaf 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-25T03:48:27,007 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/b/2a21aaf79a95477498db285aebdb2227 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732506502098/Put/seqid=0 2024-11-25T03:48:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741861_1038 (size=35082) 2024-11-25T03:48:27,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741861_1038 (size=35082) 2024-11-25T03:48:27,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741861_1038 (size=35082) 2024-11-25T03:48:27,022 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/b/2a21aaf79a95477498db285aebdb2227 2024-11-25T03:48:27,054 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/c/4a944422a1604f019ad040268eae5fa4 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732506502123/Put/seqid=0 2024-11-25T03:48:27,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741862_1039 (size=20825) 2024-11-25T03:48:27,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741862_1039 (size=20825) 2024-11-25T03:48:27,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741862_1039 (size=20825) 2024-11-25T03:48:27,064 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/c/4a944422a1604f019ad040268eae5fa4 2024-11-25T03:48:27,073 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/b/2a21aaf79a95477498db285aebdb2227 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/b/2a21aaf79a95477498db285aebdb2227 2024-11-25T03:48:27,081 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/b/2a21aaf79a95477498db285aebdb2227, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-25T03:48:27,083 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/c/4a944422a1604f019ad040268eae5fa4 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/c/4a944422a1604f019ad040268eae5fa4 2024-11-25T03:48:27,091 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/c/4a944422a1604f019ad040268eae5fa4, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-25T03:48:27,092 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1db25a25d9d8388dd3bb2b93b9e6aaaf in 93ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-25T03:48:27,103 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732506502163/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:27,106 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000003002 2024-11-25T03:48:27,106 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-25T03:48:27,107 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1db25a25d9d8388dd3bb2b93b9e6aaaf 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-25T03:48:27,118 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/c/1d057afecc5241c28b900e1e0d16e218 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732506502129/Put/seqid=0 2024-11-25T03:48:27,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741863_1040 (size=50301) 2024-11-25T03:48:27,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741863_1040 (size=50301) 2024-11-25T03:48:27,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741863_1040 (size=50301) 2024-11-25T03:48:27,127 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/c/1d057afecc5241c28b900e1e0d16e218 2024-11-25T03:48:27,135 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d057afecc5241c28b900e1e0d16e218 2024-11-25T03:48:27,136 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/.tmp/c/1d057afecc5241c28b900e1e0d16e218 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/c/1d057afecc5241c28b900e1e0d16e218 2024-11-25T03:48:27,144 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d057afecc5241c28b900e1e0d16e218 2024-11-25T03:48:27,144 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/c/1d057afecc5241c28b900e1e0d16e218, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-25T03:48:27,144 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 1db25a25d9d8388dd3bb2b93b9e6aaaf in 38ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-25T03:48:27,146 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/0000000000000003002 2024-11-25T03:48:27,147 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:27,147 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:27,148 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-25T03:48:27,150 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1db25a25d9d8388dd3bb2b93b9e6aaaf 2024-11-25T03:48:27,153 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenIntoWAL/1db25a25d9d8388dd3bb2b93b9e6aaaf/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-25T03:48:27,154 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1db25a25d9d8388dd3bb2b93b9e6aaaf; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74830941, jitterRate=0.11506791412830353}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-25T03:48:27,155 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1db25a25d9d8388dd3bb2b93b9e6aaaf: Writing region info on filesystem at 1732506506378Initializing all the Stores at 1732506506379 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506506379Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506506380 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506506380Cleaning up temporary data from old regions at 1732506507147 (+767 ms)Region opened successfully at 1732506507154 (+7 ms) 2024-11-25T03:48:27,220 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1db25a25d9d8388dd3bb2b93b9e6aaaf, disabling compactions & flushes 2024-11-25T03:48:27,220 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:27,220 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:27,220 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. after waiting 0 ms 2024-11-25T03:48:27,220 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:27,222 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732506501835.1db25a25d9d8388dd3bb2b93b9e6aaaf. 2024-11-25T03:48:27,222 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1db25a25d9d8388dd3bb2b93b9e6aaaf: Waiting for close lock at 1732506507220Disabling compacts and flushes for region at 1732506507220Disabling writes for close at 1732506507220Writing region close event to WAL at 1732506507222 (+2 ms)Closed at 1732506507222 2024-11-25T03:48:27,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741857_1034 (size=175) 2024-11-25T03:48:27,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741857_1034 (size=175) 2024-11-25T03:48:27,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741857_1034 (size=175) 2024-11-25T03:48:27,229 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:27,229 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732506506333) 2024-11-25T03:48:27,242 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 385) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_852768941_22 at /127.0.0.1:56912 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46055 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_852768941_22 at /127.0.0.1:57650 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42859 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_852768941_22 at /127.0.0.1:45660 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:46055 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:46839 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:42859 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_852768941_22 at /127.0.0.1:56902 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=438 (was 477), ProcessCount=11 (was 11), AvailableMemoryMB=8922 (was 9009) 2024-11-25T03:48:27,254 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=438, ProcessCount=11, AvailableMemoryMB=8921 2024-11-25T03:48:27,269 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:27,271 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:27,272 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:27,275 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85342216, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-85342216, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:27,288 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85342216/hregion-85342216.1732506507276, exclude list is [], retry=0 2024-11-25T03:48:27,291 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:27,291 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:27,291 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:27,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:48:27,293 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:27,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:48:27,295 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:48:27,296 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T03:48:27,296 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:27,297 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-25T03:48:27,297 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:27,321 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85342216/hregion-85342216.1732506507276 2024-11-25T03:48:27,322 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:27,322 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e88b4534d9945a106dcf91ce908c4dcf, NAME => 'test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:27,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741865_1042 (size=43) 2024-11-25T03:48:27,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741865_1042 (size=43) 2024-11-25T03:48:27,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741865_1042 (size=43) 2024-11-25T03:48:27,336 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:27,337 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,339 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88b4534d9945a106dcf91ce908c4dcf columnFamilyName a 2024-11-25T03:48:27,339 DEBUG [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:27,340 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(327): Store=e88b4534d9945a106dcf91ce908c4dcf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:27,340 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,341 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88b4534d9945a106dcf91ce908c4dcf columnFamilyName b 2024-11-25T03:48:27,342 DEBUG [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:27,342 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(327): Store=e88b4534d9945a106dcf91ce908c4dcf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:27,343 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,344 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88b4534d9945a106dcf91ce908c4dcf columnFamilyName c 2024-11-25T03:48:27,345 DEBUG [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:27,345 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(327): Store=e88b4534d9945a106dcf91ce908c4dcf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:27,346 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,346 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,347 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,348 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,348 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,349 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:27,350 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:27,352 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:27,353 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e88b4534d9945a106dcf91ce908c4dcf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60403238, jitterRate=-0.09992161393165588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:27,354 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e88b4534d9945a106dcf91ce908c4dcf: Writing region info on filesystem at 1732506507336Initializing all the Stores at 1732506507337 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506507337Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506507337Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506507337Cleaning up temporary data from old regions at 1732506507348 (+11 ms)Region opened successfully at 1732506507354 (+6 ms) 2024-11-25T03:48:27,354 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e88b4534d9945a106dcf91ce908c4dcf, disabling compactions & flushes 2024-11-25T03:48:27,354 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:27,354 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:27,354 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. after waiting 0 ms 2024-11-25T03:48:27,354 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:27,355 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:27,355 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e88b4534d9945a106dcf91ce908c4dcf: Waiting for close lock at 1732506507354Disabling compacts and flushes for region at 1732506507354Disabling writes for close at 1732506507354Writing region close event to WAL at 1732506507355 (+1 ms)Closed at 1732506507355 2024-11-25T03:48:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741864_1041 (size=175) 2024-11-25T03:48:27,358 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-85342216/hregion-85342216.1732506507276 not finished, retry = 0 2024-11-25T03:48:27,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741864_1041 (size=175) 2024-11-25T03:48:27,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741864_1041 (size=175) 2024-11-25T03:48:27,462 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:27,462 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85342216:(num 1732506507276) 2024-11-25T03:48:27,462 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:27,464 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:27,476 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464, exclude list is [], retry=0 2024-11-25T03:48:27,479 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:27,480 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:27,480 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:27,507 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 2024-11-25T03:48:27,507 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:27,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741866_1043 (size=371388) 2024-11-25T03:48:27,732 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 not finished, retry = 0 2024-11-25T03:48:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741866_1043 (size=371388) 2024-11-25T03:48:27,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741866_1043 (size=371388) 2024-11-25T03:48:27,854 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464, size=362.7 K (371388bytes) 2024-11-25T03:48:27,855 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 2024-11-25T03:48:27,855 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 after 0ms 2024-11-25T03:48:27,881 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:27,882 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 took 28ms 2024-11-25T03:48:27,886 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732506507464.temp 2024-11-25T03:48:27,912 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000000001-wal.1732506507464.temp 2024-11-25T03:48:27,929 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 so closing down 2024-11-25T03:48:27,929 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:27,933 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:27,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741867_1044 (size=371388) 2024-11-25T03:48:27,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741867_1044 (size=371388) 2024-11-25T03:48:27,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741867_1044 (size=371388) 2024-11-25T03:48:27,937 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000000001-wal.1732506507464.temp (wrote 3000 edits, skipped 0 edits in 21 ms) 2024-11-25T03:48:27,939 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000000001-wal.1732506507464.temp to hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003000 2024-11-25T03:48:27,939 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-25T03:48:27,939 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464, journal: Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464, size=362.7 K (371388bytes) at 1732506507855Split 1024 edits, skipped 0 edits. at 1732506507899 (+44 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000000001-wal.1732506507464.temp at 1732506507912 (+13 ms)Split 2048 edits, skipped 0 edits. at 1732506507915 (+3 ms)Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 so closing down at 1732506507929 (+14 ms)3 split writer threads finished at 1732506507933 (+4 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000000001-wal.1732506507464.temp (wrote 3000 edits, skipped 0 edits in 21 ms) at 1732506507937 (+4 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000000001-wal.1732506507464.temp to hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003000 at 1732506507939 (+2 ms)Processed 3000 edits across 1 Regions in 57 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732506507939 2024-11-25T03:48:27,941 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507464 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506507464 2024-11-25T03:48:27,942 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003000 2024-11-25T03:48:27,942 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:27,944 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:27,957 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944, exclude list is [], retry=0 2024-11-25T03:48:27,960 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:27,960 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:27,961 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:27,988 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944 2024-11-25T03:48:27,988 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:28,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741868_1045 (size=371515) 2024-11-25T03:48:28,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741868_1045 (size=371515) 2024-11-25T03:48:28,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741868_1045 (size=371515) 2024-11-25T03:48:28,144 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944, size=362.8 K (371515bytes) 2024-11-25T03:48:28,144 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944 2024-11-25T03:48:28,146 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944 after 1ms 2024-11-25T03:48:28,189 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:28,192 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944 took 49ms 2024-11-25T03:48:28,205 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732506507944.temp 2024-11-25T03:48:28,260 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944 so closing down 2024-11-25T03:48:28,260 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:28,271 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003001-wal.1732506507944.temp 2024-11-25T03:48:28,283 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:28,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741869_1046 (size=371515) 2024-11-25T03:48:28,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741869_1046 (size=371515) 2024-11-25T03:48:28,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741869_1046 (size=371515) 2024-11-25T03:48:28,306 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003001-wal.1732506507944.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-25T03:48:28,308 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003001-wal.1732506507944.temp to hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000006000 2024-11-25T03:48:28,308 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-25T03:48:28,309 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944, journal: Splitting hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944, size=362.8 K (371515bytes) at 1732506508144Split 1024 edits, skipped 0 edits. at 1732506508212 (+68 ms)Split 2048 edits, skipped 0 edits. at 1732506508250 (+38 ms)Finishing writing output for hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944 so closing down at 1732506508260 (+10 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003001-wal.1732506507944.temp at 1732506508271 (+11 ms)3 split writer threads finished at 1732506508283 (+12 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003001-wal.1732506507944.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1732506508306 (+23 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003001-wal.1732506507944.temp to hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000006000 at 1732506508308 (+2 ms)Processed 3000 edits across 1 Regions in 110 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732506508308 2024-11-25T03:48:28,311 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506507944 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506507944 2024-11-25T03:48:28,312 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000006000 2024-11-25T03:48:28,312 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:28,314 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/test2727-manual,16010,1732506507268, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:28,327 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506508314, exclude list is [], retry=0 2024-11-25T03:48:28,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:28,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:28,331 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:28,359 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506508314 2024-11-25T03:48:28,359 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e88b4534d9945a106dcf91ce908c4dcf, NAME => 'test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,359 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,361 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,362 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88b4534d9945a106dcf91ce908c4dcf columnFamilyName a 2024-11-25T03:48:28,362 DEBUG [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:28,363 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(327): Store=e88b4534d9945a106dcf91ce908c4dcf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:28,363 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,364 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88b4534d9945a106dcf91ce908c4dcf columnFamilyName b 2024-11-25T03:48:28,364 DEBUG [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:28,365 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(327): Store=e88b4534d9945a106dcf91ce908c4dcf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:28,365 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,367 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e88b4534d9945a106dcf91ce908c4dcf columnFamilyName c 2024-11-25T03:48:28,367 DEBUG [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:28,367 INFO [StoreOpener-e88b4534d9945a106dcf91ce908c4dcf-1 {}] regionserver.HStore(327): Store=e88b4534d9945a106dcf91ce908c4dcf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:28,368 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,371 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,372 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003000 2024-11-25T03:48:28,398 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:28,435 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003000 2024-11-25T03:48:28,436 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000006000 2024-11-25T03:48:28,464 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:28,502 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000006000 2024-11-25T03:48:28,503 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e88b4534d9945a106dcf91ce908c4dcf 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-25T03:48:28,522 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/a/2fac53d328d541feafabe1851eb52831 is 41, key is test2727/a:100/1732506507992/Put/seqid=0 2024-11-25T03:48:28,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741871_1048 (size=84227) 2024-11-25T03:48:28,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741871_1048 (size=84227) 2024-11-25T03:48:28,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741871_1048 (size=84227) 2024-11-25T03:48:28,530 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/a/2fac53d328d541feafabe1851eb52831 2024-11-25T03:48:28,563 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/b/7694d8e9ad1945d3b24fb806d943052e is 41, key is test2727/b:100/1732506508035/Put/seqid=0 2024-11-25T03:48:28,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741872_1049 (size=84609) 2024-11-25T03:48:28,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741872_1049 (size=84609) 2024-11-25T03:48:28,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741872_1049 (size=84609) 2024-11-25T03:48:28,570 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/b/7694d8e9ad1945d3b24fb806d943052e 2024-11-25T03:48:28,595 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/c/6d84375f55f946078659b8a35fc7ea04 is 41, key is test2727/c:100/1732506508077/Put/seqid=0 2024-11-25T03:48:28,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741873_1050 (size=84609) 2024-11-25T03:48:28,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741873_1050 (size=84609) 2024-11-25T03:48:28,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741873_1050 (size=84609) 2024-11-25T03:48:28,603 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/c/6d84375f55f946078659b8a35fc7ea04 2024-11-25T03:48:28,611 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/a/2fac53d328d541feafabe1851eb52831 as hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/a/2fac53d328d541feafabe1851eb52831 2024-11-25T03:48:28,618 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/a/2fac53d328d541feafabe1851eb52831, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-25T03:48:28,620 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/b/7694d8e9ad1945d3b24fb806d943052e as hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/b/7694d8e9ad1945d3b24fb806d943052e 2024-11-25T03:48:28,627 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/b/7694d8e9ad1945d3b24fb806d943052e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T03:48:28,628 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/.tmp/c/6d84375f55f946078659b8a35fc7ea04 as hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/c/6d84375f55f946078659b8a35fc7ea04 2024-11-25T03:48:28,636 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/c/6d84375f55f946078659b8a35fc7ea04, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-25T03:48:28,636 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e88b4534d9945a106dcf91ce908c4dcf in 134ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-25T03:48:28,637 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000003000 2024-11-25T03:48:28,638 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/0000000000000006000 2024-11-25T03:48:28,639 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,639 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,640 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:28,642 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e88b4534d9945a106dcf91ce908c4dcf 2024-11-25T03:48:28,645 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/test2727/e88b4534d9945a106dcf91ce908c4dcf/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-25T03:48:28,646 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e88b4534d9945a106dcf91ce908c4dcf; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74604692, jitterRate=0.11169654130935669}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:28,647 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e88b4534d9945a106dcf91ce908c4dcf: Writing region info on filesystem at 1732506508359Initializing all the Stores at 1732506508360 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506508361 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506508361Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506508361Obtaining lock to block concurrent updates at 1732506508503 (+142 ms)Preparing flush snapshotting stores in e88b4534d9945a106dcf91ce908c4dcf at 1732506508503Finished memstore snapshotting test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732506508503Flushing stores of test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. at 1732506508503Flushing e88b4534d9945a106dcf91ce908c4dcf/a: creating writer at 1732506508503Flushing e88b4534d9945a106dcf91ce908c4dcf/a: appending metadata at 1732506508521 (+18 ms)Flushing e88b4534d9945a106dcf91ce908c4dcf/a: closing flushed file at 1732506508521Flushing e88b4534d9945a106dcf91ce908c4dcf/b: creating writer at 1732506508537 (+16 ms)Flushing e88b4534d9945a106dcf91ce908c4dcf/b: appending metadata at 1732506508561 (+24 ms)Flushing e88b4534d9945a106dcf91ce908c4dcf/b: closing flushed file at 1732506508561Flushing e88b4534d9945a106dcf91ce908c4dcf/c: creating writer at 1732506508577 (+16 ms)Flushing e88b4534d9945a106dcf91ce908c4dcf/c: appending metadata at 1732506508594 (+17 ms)Flushing e88b4534d9945a106dcf91ce908c4dcf/c: closing flushed file at 1732506508594Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@492d482e: reopening flushed file at 1732506508609 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5793b199: reopening flushed file at 1732506508619 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb4735b: reopening flushed file at 1732506508627 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e88b4534d9945a106dcf91ce908c4dcf in 134ms, sequenceid=6000, compaction requested=false; wal=null at 1732506508636 (+9 ms)Cleaning up temporary data from old regions at 1732506508639 (+3 ms)Region opened successfully at 1732506508647 (+8 ms) 2024-11-25T03:48:28,649 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-25T03:48:28,649 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e88b4534d9945a106dcf91ce908c4dcf, disabling compactions & flushes 2024-11-25T03:48:28,649 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:28,649 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:28,649 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. after waiting 0 ms 2024-11-25T03:48:28,649 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:28,651 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732506507270.e88b4534d9945a106dcf91ce908c4dcf. 2024-11-25T03:48:28,651 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e88b4534d9945a106dcf91ce908c4dcf: Waiting for close lock at 1732506508649Disabling compacts and flushes for region at 1732506508649Disabling writes for close at 1732506508649Writing region close event to WAL at 1732506508651 (+2 ms)Closed at 1732506508651 2024-11-25T03:48:28,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741870_1047 (size=175) 2024-11-25T03:48:28,657 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732506507268/wal.1732506508314 not finished, retry = 0 2024-11-25T03:48:28,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741870_1047 (size=175) 2024-11-25T03:48:28,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741870_1047 (size=175) 2024-11-25T03:48:28,760 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:28,760 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732506508314) 2024-11-25T03:48:28,776 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403 (was 399) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:57024 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:45786 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:57758 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 438), ProcessCount=11 (was 11), AvailableMemoryMB=8692 (was 8921) 2024-11-25T03:48:28,788 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=8692 2024-11-25T03:48:28,811 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:28,819 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:28,820 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732506508820 2024-11-25T03:48:28,862 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820 2024-11-25T03:48:28,865 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:28,868 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8dad47b1aaa800b621a0acdd26c75621, NAME => 'testSequentialEditLogSeqNum,,1732506508812.8dad47b1aaa800b621a0acdd26c75621.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:28,868 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732506508812.8dad47b1aaa800b621a0acdd26c75621.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:28,868 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,868 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,869 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621 doesn't exist for region: 8dad47b1aaa800b621a0acdd26c75621 on table testSequentialEditLogSeqNum 2024-11-25T03:48:28,870 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 8dad47b1aaa800b621a0acdd26c75621 on table testSequentialEditLogSeqNum 2024-11-25T03:48:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741875_1052 (size=62) 2024-11-25T03:48:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741875_1052 (size=62) 2024-11-25T03:48:28,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741875_1052 (size=62) 2024-11-25T03:48:28,881 INFO [StoreOpener-8dad47b1aaa800b621a0acdd26c75621-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,883 INFO [StoreOpener-8dad47b1aaa800b621a0acdd26c75621-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8dad47b1aaa800b621a0acdd26c75621 columnFamilyName a 2024-11-25T03:48:28,883 DEBUG [StoreOpener-8dad47b1aaa800b621a0acdd26c75621-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:28,883 INFO [StoreOpener-8dad47b1aaa800b621a0acdd26c75621-1 {}] regionserver.HStore(327): Store=8dad47b1aaa800b621a0acdd26c75621/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:28,884 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,884 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,885 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,885 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,885 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,887 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8dad47b1aaa800b621a0acdd26c75621 2024-11-25T03:48:28,889 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:28,890 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8dad47b1aaa800b621a0acdd26c75621; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73256469, jitterRate=0.09160645306110382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:28,891 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8dad47b1aaa800b621a0acdd26c75621: Writing region info on filesystem at 1732506508868Initializing all the Stores at 1732506508880 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506508880Cleaning up temporary data from old regions at 1732506508885 (+5 ms)Region opened successfully at 1732506508891 (+6 ms) 2024-11-25T03:48:28,902 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8dad47b1aaa800b621a0acdd26c75621 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-25T03:48:28,923 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/.tmp/a/b6a9abe1933d4e17bf0db6a8cea39d64 is 81, key is testSequentialEditLogSeqNum/a:x0/1732506508891/Put/seqid=0 2024-11-25T03:48:28,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741876_1053 (size=5833) 2024-11-25T03:48:28,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741876_1053 (size=5833) 2024-11-25T03:48:28,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741876_1053 (size=5833) 2024-11-25T03:48:28,931 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/.tmp/a/b6a9abe1933d4e17bf0db6a8cea39d64 2024-11-25T03:48:28,938 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/.tmp/a/b6a9abe1933d4e17bf0db6a8cea39d64 as hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/a/b6a9abe1933d4e17bf0db6a8cea39d64 2024-11-25T03:48:28,945 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/a/b6a9abe1933d4e17bf0db6a8cea39d64, entries=10, sequenceid=13, filesize=5.7 K 2024-11-25T03:48:28,947 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8dad47b1aaa800b621a0acdd26c75621 in 44ms, sequenceid=13, compaction requested=false 2024-11-25T03:48:28,947 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8dad47b1aaa800b621a0acdd26c75621: 2024-11-25T03:48:28,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T03:48:28,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T03:48:28,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T03:48:28,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T03:48:28,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-25T03:48:28,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741874_1051 (size=3256) 2024-11-25T03:48:28,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741874_1051 (size=3256) 2024-11-25T03:48:28,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741874_1051 (size=3256) 2024-11-25T03:48:28,971 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820, size=3.2 K (3256bytes) 2024-11-25T03:48:28,971 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820 2024-11-25T03:48:28,972 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820 after 1ms 2024-11-25T03:48:28,999 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:28,999 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820 took 28ms 2024-11-25T03:48:29,002 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820 so closing down 2024-11-25T03:48:29,002 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:29,003 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732506508820.temp 2024-11-25T03:48:29,028 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000003-wal.1732506508820.temp 2024-11-25T03:48:29,028 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:29,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741877_1054 (size=2650) 2024-11-25T03:48:29,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741877_1054 (size=2650) 2024-11-25T03:48:29,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741877_1054 (size=2650) 2024-11-25T03:48:29,038 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000003-wal.1732506508820.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:29,039 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000003-wal.1732506508820.temp to hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000020 2024-11-25T03:48:29,039 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-25T03:48:29,039 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820, journal: Splitting hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820, size=3.2 K (3256bytes) at 1732506508971Finishing writing output for hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820 so closing down at 1732506509002 (+31 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000003-wal.1732506508820.temp at 1732506509028 (+26 ms)3 split writer threads finished at 1732506509028Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000003-wal.1732506508820.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732506509038 (+10 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000003-wal.1732506508820.temp to hdfs://localhost:46839/hbase/data/default/testSequentialEditLogSeqNum/8dad47b1aaa800b621a0acdd26c75621/recovered.edits/0000000000000000020 at 1732506509039 (+1 ms)Processed 17 edits across 1 Regions in 40 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732506508810/wal.1732506508820, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732506509039 2024-11-25T03:48:29,053 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:57758 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=8681 (was 8692) 2024-11-25T03:48:29,065 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=8679 2024-11-25T03:48:29,082 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:29,084 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:29,109 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:29,111 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65513202, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-65513202, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:29,123 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65513202/hregion-65513202.1732506509111, exclude list is [], retry=0 2024-11-25T03:48:29,126 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:29,127 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:29,127 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:29,151 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65513202/hregion-65513202.1732506509111 2024-11-25T03:48:29,151 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:29,152 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 12b41eb8002bd682ca27a8a193ba6d67, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:29,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741879_1056 (size=70) 2024-11-25T03:48:29,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741879_1056 (size=70) 2024-11-25T03:48:29,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741879_1056 (size=70) 2024-11-25T03:48:29,165 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:29,168 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,170 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName a 2024-11-25T03:48:29,170 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:29,171 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:29,171 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,173 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName b 2024-11-25T03:48:29,173 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:29,173 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:29,174 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,175 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName c 2024-11-25T03:48:29,175 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:29,176 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:29,176 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,177 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,177 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,178 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,178 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,179 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:29,180 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,182 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:29,183 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12b41eb8002bd682ca27a8a193ba6d67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67438260, jitterRate=0.004908382892608643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:29,184 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12b41eb8002bd682ca27a8a193ba6d67: Writing region info on filesystem at 1732506509165Initializing all the Stores at 1732506509166 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506509166Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506509168 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506509168Cleaning up temporary data from old regions at 1732506509178 (+10 ms)Region opened successfully at 1732506509184 (+6 ms) 2024-11-25T03:48:29,184 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 12b41eb8002bd682ca27a8a193ba6d67, disabling compactions & flushes 2024-11-25T03:48:29,184 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:29,184 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:29,184 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. after waiting 0 ms 2024-11-25T03:48:29,184 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:29,185 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:29,185 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 12b41eb8002bd682ca27a8a193ba6d67: Waiting for close lock at 1732506509184Disabling compacts and flushes for region at 1732506509184Disabling writes for close at 1732506509184Writing region close event to WAL at 1732506509185 (+1 ms)Closed at 1732506509185 2024-11-25T03:48:29,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741878_1055 (size=175) 2024-11-25T03:48:29,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741878_1055 (size=175) 2024-11-25T03:48:29,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741878_1055 (size=175) 2024-11-25T03:48:29,191 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:29,191 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65513202:(num 1732506509111) 2024-11-25T03:48:29,191 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:29,193 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:29,205 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193, exclude list is [], retry=0 2024-11-25T03:48:29,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:29,208 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:29,209 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:29,237 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 2024-11-25T03:48:29,237 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:29,237 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 12b41eb8002bd682ca27a8a193ba6d67, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:29,237 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:29,237 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,238 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,239 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,240 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName a 2024-11-25T03:48:29,240 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:29,241 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:29,241 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,242 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName b 2024-11-25T03:48:29,242 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:29,242 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:29,242 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,243 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName c 2024-11-25T03:48:29,243 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:29,244 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:29,244 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,245 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,246 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,247 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,247 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,248 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:29,250 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:29,251 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12b41eb8002bd682ca27a8a193ba6d67; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60950467, jitterRate=-0.09176726639270782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:29,252 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12b41eb8002bd682ca27a8a193ba6d67: Writing region info on filesystem at 1732506509238Initializing all the Stores at 1732506509239 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506509239Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506509239Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506509239Cleaning up temporary data from old regions at 1732506509247 (+8 ms)Region opened successfully at 1732506509252 (+5 ms) 2024-11-25T03:48:29,256 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732506509255/Put/seqid=0 2024-11-25T03:48:29,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741881_1058 (size=4826) 2024-11-25T03:48:29,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741881_1058 (size=4826) 2024-11-25T03:48:29,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741881_1058 (size=4826) 2024-11-25T03:48:29,263 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 12b41eb8002bd682ca27a8a193ba6d67/a 2024-11-25T03:48:29,272 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-25T03:48:29,273 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T03:48:29,273 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 12b41eb8002bd682ca27a8a193ba6d67: 2024-11-25T03:48:29,275 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/a/44857dd49bd447c1a11bef3b84dd8d0e_SeqId_3_ 2024-11-25T03:48:29,276 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 12b41eb8002bd682ca27a8a193ba6d67/a as hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/a/44857dd49bd447c1a11bef3b84dd8d0e_SeqId_3_ - updating store file list. 2024-11-25T03:48:29,283 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 44857dd49bd447c1a11bef3b84dd8d0e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:29,284 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/a/44857dd49bd447c1a11bef3b84dd8d0e_SeqId_3_ into 12b41eb8002bd682ca27a8a193ba6d67/a 2024-11-25T03:48:29,284 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 12b41eb8002bd682ca27a8a193ba6d67/a (new location: hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/a/44857dd49bd447c1a11bef3b84dd8d0e_SeqId_3_) 2024-11-25T03:48:29,323 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193, size=0 (0bytes) 2024-11-25T03:48:29,323 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 might be still open, length is 0 2024-11-25T03:48:29,323 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 2024-11-25T03:48:29,324 WARN [IPC Server handler 2 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-25T03:48:29,324 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 after 1ms 2024-11-25T03:48:30,995 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:45896 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45896 dst: /127.0.0.1:34347 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34347 remote=/127.0.0.1:45896]. Total timeout mills is 60000, 58295 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:30,996 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:57872 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43937:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57872 dst: /127.0.0.1:43937 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:30,996 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:57120 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33729:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57120 dst: /127.0.0.1:33729 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:30,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741880_1059 (size=606) 2024-11-25T03:48:30,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741880_1059 (size=606) 2024-11-25T03:48:33,325 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 after 4001ms 2024-11-25T03:48:33,366 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:33,366 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 took 4043ms 2024-11-25T03:48:33,370 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193; continuing. 2024-11-25T03:48:33,371 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 so closing down 2024-11-25T03:48:33,371 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:33,375 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732506509193.temp 2024-11-25T03:48:33,405 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005-wal.1732506509193.temp 2024-11-25T03:48:33,405 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:33,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741882_1060 (size=329) 2024-11-25T03:48:33,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741882_1060 (size=329) 2024-11-25T03:48:33,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741882_1060 (size=329) 2024-11-25T03:48:33,415 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005-wal.1732506509193.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:33,416 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005-wal.1732506509193.temp to hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005 2024-11-25T03:48:33,416 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 48 ms; skipped=1; WAL=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T03:48:33,417 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193, journal: Splitting hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193, size=0 (0bytes) at 1732506509323Finishing writing output for hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 so closing down at 1732506513371 (+4048 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005-wal.1732506509193.temp at 1732506513405 (+34 ms)3 split writer threads finished at 1732506513405Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005-wal.1732506509193.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732506513415 (+10 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005-wal.1732506509193.temp to hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005 at 1732506513416 (+1 ms)Processed 2 edits across 1 Regions in 48 ms; skipped=1; WAL=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193, size=0, length=0, corrupted=false, cancelled=false at 1732506513417 (+1 ms) 2024-11-25T03:48:33,418 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506509193 2024-11-25T03:48:33,419 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005 2024-11-25T03:48:33,420 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:33,422 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:33,435 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506513422, exclude list is [], retry=0 2024-11-25T03:48:33,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:33,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:33,440 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:33,483 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506513422 2024-11-25T03:48:33,483 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:33,483 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 12b41eb8002bd682ca27a8a193ba6d67, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:33,484 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:33,484 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,484 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,486 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,486 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName a 2024-11-25T03:48:33,487 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:33,496 DEBUG [StoreFileOpener-12b41eb8002bd682ca27a8a193ba6d67-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 44857dd49bd447c1a11bef3b84dd8d0e_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:33,496 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/a/44857dd49bd447c1a11bef3b84dd8d0e_SeqId_3_ 2024-11-25T03:48:33,496 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:33,496 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,497 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName b 2024-11-25T03:48:33,498 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:33,498 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:33,499 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,500 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12b41eb8002bd682ca27a8a193ba6d67 columnFamilyName c 2024-11-25T03:48:33,501 DEBUG [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:33,501 INFO [StoreOpener-12b41eb8002bd682ca27a8a193ba6d67-1 {}] regionserver.HStore(327): Store=12b41eb8002bd682ca27a8a193ba6d67/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:33,501 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,502 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,504 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,504 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005 2024-11-25T03:48:33,535 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:33,537 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005 2024-11-25T03:48:33,537 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 12b41eb8002bd682ca27a8a193ba6d67 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-25T03:48:33,556 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/.tmp/a/5c695295dd5944acb1d53f585c49936b is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732506509290/Put/seqid=0 2024-11-25T03:48:33,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741884_1062 (size=5149) 2024-11-25T03:48:33,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741884_1062 (size=5149) 2024-11-25T03:48:33,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741884_1062 (size=5149) 2024-11-25T03:48:33,567 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/.tmp/a/5c695295dd5944acb1d53f585c49936b 2024-11-25T03:48:33,575 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/.tmp/a/5c695295dd5944acb1d53f585c49936b as hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/a/5c695295dd5944acb1d53f585c49936b 2024-11-25T03:48:33,587 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/a/5c695295dd5944acb1d53f585c49936b, entries=1, sequenceid=5, filesize=5.0 K 2024-11-25T03:48:33,588 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 12b41eb8002bd682ca27a8a193ba6d67 in 51ms, sequenceid=5, compaction requested=false; wal=null 2024-11-25T03:48:33,589 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/0000000000000000005 2024-11-25T03:48:33,590 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,590 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,591 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:33,592 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 12b41eb8002bd682ca27a8a193ba6d67 2024-11-25T03:48:33,594 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/12b41eb8002bd682ca27a8a193ba6d67/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-25T03:48:33,596 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 12b41eb8002bd682ca27a8a193ba6d67; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63978178, jitterRate=-0.046650856733322144}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:33,597 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 12b41eb8002bd682ca27a8a193ba6d67: Writing region info on filesystem at 1732506513484Initializing all the Stores at 1732506513485 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506513485Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506513485Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506513485Obtaining lock to block concurrent updates at 1732506513537 (+52 ms)Preparing flush snapshotting stores in 12b41eb8002bd682ca27a8a193ba6d67 at 1732506513537Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732506513537Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. at 1732506513537Flushing 12b41eb8002bd682ca27a8a193ba6d67/a: creating writer at 1732506513537Flushing 12b41eb8002bd682ca27a8a193ba6d67/a: appending metadata at 1732506513556 (+19 ms)Flushing 12b41eb8002bd682ca27a8a193ba6d67/a: closing flushed file at 1732506513556Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57088b31: reopening flushed file at 1732506513574 (+18 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 12b41eb8002bd682ca27a8a193ba6d67 in 51ms, sequenceid=5, compaction requested=false; wal=null at 1732506513588 (+14 ms)Cleaning up temporary data from old regions at 1732506513590 (+2 ms)Region opened successfully at 1732506513597 (+7 ms) 2024-11-25T03:48:33,601 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 12b41eb8002bd682ca27a8a193ba6d67, disabling compactions & flushes 2024-11-25T03:48:33,602 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:33,602 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:33,602 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. after waiting 0 ms 2024-11-25T03:48:33,602 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:33,605 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732506509083.12b41eb8002bd682ca27a8a193ba6d67. 2024-11-25T03:48:33,605 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 12b41eb8002bd682ca27a8a193ba6d67: Waiting for close lock at 1732506513601Disabling compacts and flushes for region at 1732506513601Disabling writes for close at 1732506513602 (+1 ms)Writing region close event to WAL at 1732506513605 (+3 ms)Closed at 1732506513605 2024-11-25T03:48:33,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741883_1061 (size=175) 2024-11-25T03:48:33,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741883_1061 (size=175) 2024-11-25T03:48:33,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741883_1061 (size=175) 2024-11-25T03:48:33,611 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:33,611 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732506513422) 2024-11-25T03:48:33,630 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=411 (was 408) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_511884881_22 at /127.0.0.1:48316 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_511884881_22 at /127.0.0.1:39984 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:46839 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_511884881_22 at /127.0.0.1:55704 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=411 (was 403) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8673 (was 8679) 2024-11-25T03:48:33,644 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=411, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=411, ProcessCount=11, AvailableMemoryMB=8672 2024-11-25T03:48:33,665 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:33,672 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T03:48:33,677 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is d73269e94d72,41803,1732506496940 2024-11-25T03:48:33,679 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d19f176 2024-11-25T03:48:33,680 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T03:48:33,683 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T03:48:33,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T03:48:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T03:48:33,698 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T03:48:33,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-25T03:48:33,701 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:33,705 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T03:48:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:48:33,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741885_1063 (size=694) 2024-11-25T03:48:33,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741885_1063 (size=694) 2024-11-25T03:48:33,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741885_1063 (size=694) 2024-11-25T03:48:33,757 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1b485cca92b597d29ec36b9eadec9462, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2 2024-11-25T03:48:33,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741886_1064 (size=77) 2024-11-25T03:48:33,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741886_1064 (size=77) 2024-11-25T03:48:33,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741886_1064 (size=77) 2024-11-25T03:48:33,780 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:33,780 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 1b485cca92b597d29ec36b9eadec9462, disabling compactions & flushes 2024-11-25T03:48:33,781 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:33,781 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:33,781 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. after waiting 0 ms 2024-11-25T03:48:33,781 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:33,781 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:33,781 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1b485cca92b597d29ec36b9eadec9462: Waiting for close lock at 1732506513780Disabling compacts and flushes for region at 1732506513780Disabling writes for close at 1732506513781 (+1 ms)Writing region close event to WAL at 1732506513781Closed at 1732506513781 2024-11-25T03:48:33,783 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T03:48:33,787 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732506513783"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732506513783"}]},"ts":"1732506513783"} 2024-11-25T03:48:33,794 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T03:48:33,796 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T03:48:33,800 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506513796"}]},"ts":"1732506513796"} 2024-11-25T03:48:33,807 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-25T03:48:33,808 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:48:33,810 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:48:33,810 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:48:33,810 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-25T03:48:33,810 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:48:33,810 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:48:33,810 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-25T03:48:33,810 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:48:33,810 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:48:33,810 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-25T03:48:33,810 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:48:33,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN}] 2024-11-25T03:48:33,815 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN 2024-11-25T03:48:33,818 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN; state=OFFLINE, location=d73269e94d72,33219,1732506497809; forceNewPlan=false, retain=false 2024-11-25T03:48:33,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:48:33,971 INFO [d73269e94d72:41803 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T03:48:33,972 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPENING, regionLocation=d73269e94d72,33219,1732506497809 2024-11-25T03:48:33,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN because future has completed 2024-11-25T03:48:33,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809}] 2024-11-25T03:48:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:48:34,136 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b485cca92b597d29ec36b9eadec9462, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:34,136 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,137 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:34,137 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,137 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,139 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,140 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf1 2024-11-25T03:48:34,141 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:34,141 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:34,141 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,143 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf2 2024-11-25T03:48:34,143 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:34,143 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:34,144 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,144 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,145 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,146 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,146 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,148 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T03:48:34,150 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,152 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:34,152 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1b485cca92b597d29ec36b9eadec9462; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63884228, jitterRate=-0.04805082082748413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T03:48:34,153 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,154 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1b485cca92b597d29ec36b9eadec9462: Running coprocessor pre-open hook at 1732506514137Writing region info on filesystem at 1732506514137Initializing all the Stores at 1732506514138 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506514138Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506514139 (+1 ms)Cleaning up temporary data from old regions at 1732506514146 (+7 ms)Running coprocessor post-open hooks at 1732506514153 (+7 ms)Region opened successfully at 1732506514153 2024-11-25T03:48:34,156 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., pid=6, masterSystemTime=1732506514130 2024-11-25T03:48:34,159 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,159 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,160 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPEN, openSeqNum=2, regionLocation=d73269e94d72,33219,1732506497809 2024-11-25T03:48:34,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809 because future has completed 2024-11-25T03:48:34,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T03:48:34,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809 in 190 msec 2024-11-25T03:48:34,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T03:48:34,190 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T03:48:34,190 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN in 365 msec 2024-11-25T03:48:34,190 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732506514190"}]},"ts":"1732506514190"} 2024-11-25T03:48:34,194 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-25T03:48:34,195 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T03:48:34,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 506 msec 2024-11-25T03:48:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T03:48:34,341 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-25T03:48:34,341 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-25T03:48:34,343 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:48:34,349 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-25T03:48:34,349 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T03:48:34,350 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-25T03:48:34,365 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=2] 2024-11-25T03:48:34,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1b485cca92b597d29ec36b9eadec9462, source=d73269e94d72,33219,1732506497809, destination=d73269e94d72,33171,1732506497711, warming up region on d73269e94d72,33171,1732506497711 2024-11-25T03:48:34,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:48:34,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1b485cca92b597d29ec36b9eadec9462, source=d73269e94d72,33219,1732506497809, destination=d73269e94d72,33171,1732506497711, running balancer 2024-11-25T03:48:34,388 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41401, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:48:34,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE 2024-11-25T03:48:34,389 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE 2024-11-25T03:48:34,392 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=CLOSING, regionLocation=d73269e94d72,33219,1732506497809 2024-11-25T03:48:34,392 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=d73269e94d72,33219,1732506497809, table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:48:34,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE because future has completed 2024-11-25T03:48:34,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1b485cca92b597d29ec36b9eadec9462, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:34,402 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:48:34,403 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809}] 2024-11-25T03:48:34,407 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,409 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf1 2024-11-25T03:48:34,409 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:34,411 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:34,412 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,414 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf2 2024-11-25T03:48:34,414 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:34,414 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(1722): Closing 1b485cca92b597d29ec36b9eadec9462, disabling compactions & flushes 2024-11-25T03:48:34,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. after waiting 0 ms 2024-11-25T03:48:34,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,416 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] regionserver.HRegion(1676): Region close journal for 1b485cca92b597d29ec36b9eadec9462: Waiting for close lock at 1732506514415Disabling compacts and flushes for region at 1732506514415Disabling writes for close at 1732506514415Writing region close event to WAL at 1732506514416 (+1 ms)Closed at 1732506514416 2024-11-25T03:48:34,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T03:48:34,563 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,563 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:48:34,564 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 1b485cca92b597d29ec36b9eadec9462, disabling compactions & flushes 2024-11-25T03:48:34,564 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,564 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,564 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. after waiting 0 ms 2024-11-25T03:48:34,564 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,565 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 1b485cca92b597d29ec36b9eadec9462 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-25T03:48:34,593 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf is 35, key is r1/cf1:q/1732506514368/Put/seqid=0 2024-11-25T03:48:34,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741887_1065 (size=4783) 2024-11-25T03:48:34,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741887_1065 (size=4783) 2024-11-25T03:48:34,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741887_1065 (size=4783) 2024-11-25T03:48:34,621 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf 2024-11-25T03:48:34,633 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf 2024-11-25T03:48:34,648 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf, entries=1, sequenceid=5, filesize=4.7 K 2024-11-25T03:48:34,651 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1b485cca92b597d29ec36b9eadec9462 in 87ms, sequenceid=5, compaction requested=false 2024-11-25T03:48:34,651 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-25T03:48:34,675 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T03:48:34,680 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:34,681 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 1b485cca92b597d29ec36b9eadec9462: Waiting for close lock at 1732506514564Running coprocessor pre-close hooks at 1732506514564Disabling compacts and flushes for region at 1732506514564Disabling writes for close at 1732506514564Obtaining lock to block concurrent updates at 1732506514565 (+1 ms)Preparing flush snapshotting stores in 1b485cca92b597d29ec36b9eadec9462 at 1732506514565Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732506514566 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. at 1732506514566Flushing 1b485cca92b597d29ec36b9eadec9462/cf1: creating writer at 1732506514567 (+1 ms)Flushing 1b485cca92b597d29ec36b9eadec9462/cf1: appending metadata at 1732506514593 (+26 ms)Flushing 1b485cca92b597d29ec36b9eadec9462/cf1: closing flushed file at 1732506514593Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a5b5308: reopening flushed file at 1732506514631 (+38 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 1b485cca92b597d29ec36b9eadec9462 in 87ms, sequenceid=5, compaction requested=false at 1732506514651 (+20 ms)Writing region close event to WAL at 1732506514663 (+12 ms)Running coprocessor post-close hooks at 1732506514678 (+15 ms)Closed at 1732506514680 (+2 ms) 2024-11-25T03:48:34,681 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 1b485cca92b597d29ec36b9eadec9462 move to d73269e94d72,33171,1732506497711 record at close sequenceid=5 2024-11-25T03:48:34,687 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:34,688 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=CLOSED 2024-11-25T03:48:34,691 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809 because future has completed 2024-11-25T03:48:34,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T03:48:34,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809 in 295 msec 2024-11-25T03:48:34,703 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE; state=CLOSED, location=d73269e94d72,33171,1732506497711; forceNewPlan=false, retain=false 2024-11-25T03:48:34,854 INFO [d73269e94d72:41803 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T03:48:34,854 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPENING, regionLocation=d73269e94d72,33171,1732506497711 2024-11-25T03:48:34,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE because future has completed 2024-11-25T03:48:34,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711}] 2024-11-25T03:48:35,029 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,029 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b485cca92b597d29ec36b9eadec9462, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:35,030 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,030 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:35,030 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,030 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,039 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,041 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf1 2024-11-25T03:48:35,041 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:35,111 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf 2024-11-25T03:48:35,111 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:35,112 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,115 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf2 2024-11-25T03:48:35,115 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:35,123 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:35,124 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,125 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,127 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,128 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,128 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,129 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T03:48:35,131 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,132 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1b485cca92b597d29ec36b9eadec9462; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70330530, jitterRate=0.0480065643787384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T03:48:35,133 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,133 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1b485cca92b597d29ec36b9eadec9462: Running coprocessor pre-open hook at 1732506515031Writing region info on filesystem at 1732506515031Initializing all the Stores at 1732506515038 (+7 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506515038Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506515039 (+1 ms)Cleaning up temporary data from old regions at 1732506515128 (+89 ms)Running coprocessor post-open hooks at 1732506515133 (+5 ms)Region opened successfully at 1732506515133 2024-11-25T03:48:35,135 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., pid=9, masterSystemTime=1732506515023 2024-11-25T03:48:35,140 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,141 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,141 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPEN, openSeqNum=9, regionLocation=d73269e94d72,33171,1732506497711 2024-11-25T03:48:35,146 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711 because future has completed 2024-11-25T03:48:35,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T03:48:35,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711 in 288 msec 2024-11-25T03:48:35,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE in 773 msec 2024-11-25T03:48:35,197 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:48:35,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:48:35,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:45838 deadline: 1732506575205, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33171 startCode=1732506497711. As of locationSeqNum=5. 2024-11-25T03:48:35,237 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33171 startCode=1732506497711. As of locationSeqNum=5. 2024-11-25T03:48:35,238 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33171 startCode=1732506497711. As of locationSeqNum=5. 2024-11-25T03:48:35,238 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33171,1732506497711, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33171 startCode=1732506497711. As of locationSeqNum=5. 2024-11-25T03:48:35,350 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:48:35,352 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:48:35,363 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1b485cca92b597d29ec36b9eadec9462 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-25T03:48:35,397 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf1/c7ed905e9e59486bb47a4b95409c8f79 is 29, key is r1/cf1:/1732506515353/DeleteFamily/seqid=0 2024-11-25T03:48:35,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741888_1066 (size=4906) 2024-11-25T03:48:35,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741888_1066 (size=4906) 2024-11-25T03:48:35,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741888_1066 (size=4906) 2024-11-25T03:48:35,445 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf1/c7ed905e9e59486bb47a4b95409c8f79 2024-11-25T03:48:35,458 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7ed905e9e59486bb47a4b95409c8f79 2024-11-25T03:48:35,480 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf2/2bce3d99f0a2497c9371376e972b65b3 is 29, key is r1/cf2:/1732506515353/DeleteFamily/seqid=0 2024-11-25T03:48:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741889_1067 (size=4906) 2024-11-25T03:48:35,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741889_1067 (size=4906) 2024-11-25T03:48:35,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741889_1067 (size=4906) 2024-11-25T03:48:35,495 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf2/2bce3d99f0a2497c9371376e972b65b3 2024-11-25T03:48:35,508 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bce3d99f0a2497c9371376e972b65b3 2024-11-25T03:48:35,512 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf1/c7ed905e9e59486bb47a4b95409c8f79 as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/c7ed905e9e59486bb47a4b95409c8f79 2024-11-25T03:48:35,521 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7ed905e9e59486bb47a4b95409c8f79 2024-11-25T03:48:35,522 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/c7ed905e9e59486bb47a4b95409c8f79, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T03:48:35,523 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf2/2bce3d99f0a2497c9371376e972b65b3 as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/2bce3d99f0a2497c9371376e972b65b3 2024-11-25T03:48:35,533 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bce3d99f0a2497c9371376e972b65b3 2024-11-25T03:48:35,533 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/2bce3d99f0a2497c9371376e972b65b3, entries=1, sequenceid=12, filesize=4.8 K 2024-11-25T03:48:35,535 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 1b485cca92b597d29ec36b9eadec9462 in 172ms, sequenceid=12, compaction requested=false 2024-11-25T03:48:35,535 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1b485cca92b597d29ec36b9eadec9462: 2024-11-25T03:48:35,538 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-25T03:48:35,540 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1b485cca92b597d29ec36b9eadec9462/cf1 is initiating major compaction (all files) 2024-11-25T03:48:35,540 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:48:35,540 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:35,541 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1b485cca92b597d29ec36b9eadec9462/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,542 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf, hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/c7ed905e9e59486bb47a4b95409c8f79] into tmpdir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp, totalSize=9.5 K 2024-11-25T03:48:35,543 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ee7f9e58d9da4ba19ce6ed30870e2edf, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732506514368 2024-11-25T03:48:35,544 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c7ed905e9e59486bb47a4b95409c8f79, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T03:48:35,560 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1b485cca92b597d29ec36b9eadec9462#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:48:35,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741890_1068 (size=4626) 2024-11-25T03:48:35,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741890_1068 (size=4626) 2024-11-25T03:48:35,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741890_1068 (size=4626) 2024-11-25T03:48:35,583 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf1/1947dbdd0b6d4b719d1d9c8446b811ee as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/1947dbdd0b6d4b719d1d9c8446b811ee 2024-11-25T03:48:35,609 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 1b485cca92b597d29ec36b9eadec9462/cf1 of 1b485cca92b597d29ec36b9eadec9462 into 1947dbdd0b6d4b719d1d9c8446b811ee(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:48:35,609 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1b485cca92b597d29ec36b9eadec9462: 2024-11-25T03:48:35,609 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-25T03:48:35,610 DEBUG [Time-limited test {}] regionserver.HStore(1541): 1b485cca92b597d29ec36b9eadec9462/cf2 is initiating major compaction (all files) 2024-11-25T03:48:35,610 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T03:48:35,610 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T03:48:35,610 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 1b485cca92b597d29ec36b9eadec9462/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,612 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/2bce3d99f0a2497c9371376e972b65b3] into tmpdir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp, totalSize=4.8 K 2024-11-25T03:48:35,614 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2bce3d99f0a2497c9371376e972b65b3, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-25T03:48:35,628 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 1b485cca92b597d29ec36b9eadec9462#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-25T03:48:35,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741891_1069 (size=4592) 2024-11-25T03:48:35,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741891_1069 (size=4592) 2024-11-25T03:48:35,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741891_1069 (size=4592) 2024-11-25T03:48:35,659 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/.tmp/cf2/118c813ea5c443ceb16bb916d2a7598a as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/118c813ea5c443ceb16bb916d2a7598a 2024-11-25T03:48:35,672 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 1b485cca92b597d29ec36b9eadec9462/cf2 of 1b485cca92b597d29ec36b9eadec9462 into 118c813ea5c443ceb16bb916d2a7598a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:48:35,672 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 1b485cca92b597d29ec36b9eadec9462: 2024-11-25T03:48:35,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=1b485cca92b597d29ec36b9eadec9462, source=d73269e94d72,33171,1732506497711, destination=d73269e94d72,33219,1732506497809, warming up region on d73269e94d72,33219,1732506497809 2024-11-25T03:48:35,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=1b485cca92b597d29ec36b9eadec9462, source=d73269e94d72,33171,1732506497711, destination=d73269e94d72,33219,1732506497809, running balancer 2024-11-25T03:48:35,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE 2024-11-25T03:48:35,681 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE 2024-11-25T03:48:35,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=CLOSING, regionLocation=d73269e94d72,33171,1732506497711 2024-11-25T03:48:35,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(7855): Warmup {ENCODED => 1b485cca92b597d29ec36b9eadec9462, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:35,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:35,685 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,688 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf1 2024-11-25T03:48:35,688 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:35,689 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE because future has completed 2024-11-25T03:48:35,696 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-25T03:48:35,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711}] 2024-11-25T03:48:35,712 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/1947dbdd0b6d4b719d1d9c8446b811ee 2024-11-25T03:48:35,723 INFO [StoreFileOpener-1b485cca92b597d29ec36b9eadec9462-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c7ed905e9e59486bb47a4b95409c8f79 2024-11-25T03:48:35,724 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/c7ed905e9e59486bb47a4b95409c8f79 2024-11-25T03:48:35,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-25T03:48:35,735 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf 2024-11-25T03:48:35,735 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:35,736 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,737 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf2 2024-11-25T03:48:35,738 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:35,752 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/118c813ea5c443ceb16bb916d2a7598a 2024-11-25T03:48:35,765 INFO [StoreFileOpener-1b485cca92b597d29ec36b9eadec9462-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2bce3d99f0a2497c9371376e972b65b3 2024-11-25T03:48:35,766 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/2bce3d99f0a2497c9371376e972b65b3 2024-11-25T03:48:35,766 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(1722): Closing 1b485cca92b597d29ec36b9eadec9462, disabling compactions & flushes 2024-11-25T03:48:35,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. after waiting 0 ms 2024-11-25T03:48:35,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33219 {}] regionserver.HRegion(1676): Region close journal for 1b485cca92b597d29ec36b9eadec9462: Waiting for close lock at 1732506515767Disabling compacts and flushes for region at 1732506515767Disabling writes for close at 1732506515767Writing region close event to WAL at 1732506515772 (+5 ms)Closed at 1732506515772 2024-11-25T03:48:35,853 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,853 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-25T03:48:35,854 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 1b485cca92b597d29ec36b9eadec9462, disabling compactions & flushes 2024-11-25T03:48:35,854 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,854 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,854 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. after waiting 0 ms 2024-11-25T03:48:35,854 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,854 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf, hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/c7ed905e9e59486bb47a4b95409c8f79] to archive 2024-11-25T03:48:35,859 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:48:35,868 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/ee7f9e58d9da4ba19ce6ed30870e2edf 2024-11-25T03:48:35,870 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/c7ed905e9e59486bb47a4b95409c8f79 to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/c7ed905e9e59486bb47a4b95409c8f79 2024-11-25T03:48:35,881 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/2bce3d99f0a2497c9371376e972b65b3] to archive 2024-11-25T03:48:35,882 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:48:35,884 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/2bce3d99f0a2497c9371376e972b65b3 to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/2bce3d99f0a2497c9371376e972b65b3 2024-11-25T03:48:35,889 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-25T03:48:35,890 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:35,890 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 1b485cca92b597d29ec36b9eadec9462: Waiting for close lock at 1732506515854Running coprocessor pre-close hooks at 1732506515854Disabling compacts and flushes for region at 1732506515854Disabling writes for close at 1732506515854Writing region close event to WAL at 1732506515885 (+31 ms)Running coprocessor post-close hooks at 1732506515890 (+5 ms)Closed at 1732506515890 2024-11-25T03:48:35,890 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 1b485cca92b597d29ec36b9eadec9462 move to d73269e94d72,33219,1732506497809 record at close sequenceid=12 2024-11-25T03:48:35,893 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:35,895 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=CLOSED 2024-11-25T03:48:35,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711 because future has completed 2024-11-25T03:48:35,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T03:48:35,907 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711 in 207 msec 2024-11-25T03:48:35,908 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE; state=CLOSED, location=d73269e94d72,33219,1732506497809; forceNewPlan=false, retain=false 2024-11-25T03:48:36,059 INFO [d73269e94d72:41803 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T03:48:36,059 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPENING, regionLocation=d73269e94d72,33219,1732506497809 2024-11-25T03:48:36,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE because future has completed 2024-11-25T03:48:36,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809}] 2024-11-25T03:48:36,224 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,224 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b485cca92b597d29ec36b9eadec9462, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:36,225 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,225 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:36,225 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,225 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,228 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,229 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf1 2024-11-25T03:48:36,230 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:36,244 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/1947dbdd0b6d4b719d1d9c8446b811ee 2024-11-25T03:48:36,244 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:36,244 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,247 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf2 2024-11-25T03:48:36,248 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:36,264 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/118c813ea5c443ceb16bb916d2a7598a 2024-11-25T03:48:36,265 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:36,265 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,267 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,270 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,271 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,271 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,272 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T03:48:36,274 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,276 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 1b485cca92b597d29ec36b9eadec9462; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62261619, jitterRate=-0.07222957909107208}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T03:48:36,277 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,278 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 1b485cca92b597d29ec36b9eadec9462: Running coprocessor pre-open hook at 1732506516225Writing region info on filesystem at 1732506516225Initializing all the Stores at 1732506516228 (+3 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506516228Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506516228Cleaning up temporary data from old regions at 1732506516271 (+43 ms)Running coprocessor post-open hooks at 1732506516277 (+6 ms)Region opened successfully at 1732506516278 (+1 ms) 2024-11-25T03:48:36,280 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., pid=12, masterSystemTime=1732506516218 2024-11-25T03:48:36,286 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,287 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPEN, openSeqNum=18, regionLocation=d73269e94d72,33219,1732506497809 2024-11-25T03:48:36,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809 because future has completed 2024-11-25T03:48:36,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-25T03:48:36,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33219,1732506497809 in 231 msec 2024-11-25T03:48:36,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, REOPEN/MOVE in 618 msec 2024-11-25T03:48:36,333 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:48:36,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:48:36,338 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server d73269e94d72,33219,1732506497809: testing ***** 2024-11-25T03:48:36,338 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-25T03:48:36,340 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-25T03:48:36,341 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-25T03:48:36,350 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-25T03:48:36,352 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-25T03:48:36,374 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 478118368 }, "NonHeapMemoryUsage": { "committed": 171638784, "init": 7667712, "max": -1, "used": 169063648 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "d73269e94d72", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10264, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 93, "ProcessCallTime_mean": 6, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 4, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 27, "ProcessCallTime_98th_percentile": 93, "ProcessCallTime_99th_percentile": 93, "ProcessCallTime_99.9th_percentile": 93, "ProcessCallTime_TimeRangeCount_0-1": 6, "ProcessCallTime_TimeRangeCount_1-3": 15, "ProcessCallTime_TimeRangeCount_3-10": 8, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 93, "TotalCallTime_mean": 6, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 28, "TotalCallTime_98th_percentile": 93, "TotalCallTime_99th_percentile": 93, "TotalCallTime_99.9th_percentile": 93, "TotalCallTime_TimeRangeCount_0-1": 6, "TotalCallTime_TimeRangeCount_1-3": 15, "TotalCallTime_TimeRangeCount_3-10": 8, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5773 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "d73269e94d72", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57852", "tag.serverName": "d73269e94d72,33219,1732506497809", "tag.clusterId": "26f49762-00a5-45ef-9268-18b170fa2906", "tag.Context": "regionserver", "tag.Hostname": "d73269e94d72", "regionCount": 2, "storeCount": 6, "hlogFileCount": 6, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 1767, "memStoreHeapSize": 4376, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732506497809, "averageRegionSize": 883, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 1.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 599040, "ByteBuffAllocatorHeapAllocationRatio": 0.35775729671400613, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 29, "totalRowActionRequestCount": 14, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 7, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 17, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 1, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "ScanTime_TimeRangeCount_1-3": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 16, "Put_min": 1, "Put_max": 4, "Put_mean": 2, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 2, "Put_90th_percentile": 3, "Put_95th_percentile": 4, "Put_98th_percentile": 4, "Put_99th_percentile": 4, "Put_99.9th_percentile": 4, "Put_TimeRangeCount_0-1": 2, "Put_TimeRangeCount_1-3": 11, "Put_TimeRangeCount_3-10": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 2, "Get_min": 1, "Get_max": 1, "Get_mean": 1, "Get_25th_percentile": 1, "Get_median": 1, "Get_75th_percentile": 1, "Get_90th_percentile": 1, "Get_95th_percentile": 1, "Get_98th_percentile": 1, "Get_99th_percentile": 1, "Get_99.9th_percentile": 1, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 87, "FlushTime_max": 87, "FlushTime_mean": 87, "FlushTime_25th_percentile": 87, "FlushTime_median": 87, "FlushTime_75th_percentile": 87, "FlushTime_90th_percentile": 87, "FlushTime_95th_percentile": 87, "FlushTime_98th_percentile": 87, "FlushTime_99th_percentile": 87, "FlushTime_99.9th_percentile": 87, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 64, "ScanSize_max": 1112, "ScanSize_mean": 902, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_10-100": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-25T03:48:36,377 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41803 {}] master.MasterRpcServices(700): d73269e94d72,33219,1732506497809 reported a fatal error: ***** ABORTING region server d73269e94d72,33219,1732506497809: testing ***** 2024-11-25T03:48:36,379 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,33219,1732506497809' ***** 2024-11-25T03:48:36,379 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-25T03:48:36,380 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:48:36,380 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:48:36,380 INFO [RS:1;d73269e94d72:33219 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-25T03:48:36,381 INFO [RS:1;d73269e94d72:33219 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-25T03:48:36,381 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33171 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:43056 deadline: 1732506576380, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33219 startCode=1732506497809. As of locationSeqNum=12. 2024-11-25T03:48:36,381 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(956): aborting server d73269e94d72,33219,1732506497809 2024-11-25T03:48:36,381 INFO [RS:1;d73269e94d72:33219 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:48:36,382 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b485cca92b597d29ec36b9eadec9462, disabling compactions & flushes 2024-11-25T03:48:36,382 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,382 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,382 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. after waiting 0 ms 2024-11-25T03:48:36,382 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,382 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33171,1732506497711, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33171,1732506497711, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33219 startCode=1732506497809. As of locationSeqNum=12. 2024-11-25T03:48:36,382 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33171,1732506497711, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33219 startCode=1732506497809. As of locationSeqNum=12. 2024-11-25T03:48:36,382 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33171,1732506497711, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=d73269e94d72 port=33219 startCode=1732506497809. As of locationSeqNum=12. 2024-11-25T03:48:36,382 INFO [RS:1;d73269e94d72:33219 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;d73269e94d72:33219. 2024-11-25T03:48:36,382 DEBUG [RS:1;d73269e94d72:33219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:48:36,383 DEBUG [RS:1;d73269e94d72:33219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:48:36,384 INFO [RS:1;d73269e94d72:33219 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:48:36,384 INFO [RS:1;d73269e94d72:33219 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:48:36,384 INFO [RS:1;d73269e94d72:33219 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:48:36,384 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:48:36,385 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d73269e94d72,33219,1732506497809 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:36,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d73269e94d72,33219,1732506497809 aborting 2024-11-25T03:48:36,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d73269e94d72,33219,1732506497809 aborting 2024-11-25T03:48:36,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 from cache 2024-11-25T03:48:36,389 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T03:48:36,389 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1325): Online Regions={1b485cca92b597d29ec36b9eadec9462=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., 1588230740=hbase:meta,,1.1588230740} 2024-11-25T03:48:36,389 DEBUG [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:36,390 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,390 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:48:36,390 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b485cca92b597d29ec36b9eadec9462: Waiting for close lock at 1732506516381Running coprocessor pre-close hooks at 1732506516382 (+1 ms)Disabling compacts and flushes for region at 1732506516382Disabling writes for close at 1732506516382Writing region close event to WAL at 1732506516389 (+7 ms)Running coprocessor post-close hooks at 1732506516389Closed at 1732506516389 2024-11-25T03:48:36,390 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:48:36,390 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:48:36,390 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:36,390 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:48:36,390 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:48:36,396 ERROR [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-25T03:48:36,397 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:48:36,397 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:48:36,397 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506516390Running coprocessor pre-close hooks at 1732506516390Disabling compacts and flushes for region at 1732506516390Disabling writes for close at 1732506516390Writing region close event to WAL at 1732506516396 (+6 ms)Running coprocessor post-close hooks at 1732506516396Closed at 1732506516397 (+1 ms) 2024-11-25T03:48:36,397 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:48:36,423 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:48:36,489 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:36,491 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1] 2024-11-25T03:48:36,491 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d73269e94d72,33219,1732506497809 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:36,492 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d73269e94d72,33219,1732506497809 aborting 2024-11-25T03:48:36,492 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server d73269e94d72,33219,1732506497809 aborting 2024-11-25T03:48:36,492 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 from cache 2024-11-25T03:48:36,590 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,33219,1732506497809; all regions closed. 2024-11-25T03:48:36,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741836_1012 (size=7323) 2024-11-25T03:48:36,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741836_1012 (size=7323) 2024-11-25T03:48:36,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741836_1012 (size=7323) 2024-11-25T03:48:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741835_1011 (size=2288) 2024-11-25T03:48:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741835_1011 (size=2288) 2024-11-25T03:48:36,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741835_1011 (size=2288) 2024-11-25T03:48:36,612 DEBUG [RS:1;d73269e94d72:33219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:48:36,613 INFO [RS:1;d73269e94d72:33219 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:48:36,613 INFO [RS:1;d73269e94d72:33219 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:48:36,613 INFO [RS:1;d73269e94d72:33219 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T03:48:36,613 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:48:36,613 INFO [RS:1;d73269e94d72:33219 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:48:36,614 INFO [RS:1;d73269e94d72:33219 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33219 2024-11-25T03:48:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,33219,1732506497809 2024-11-25T03:48:36,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:48:36,673 INFO [RS:1;d73269e94d72:33219 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:48:36,674 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,33219,1732506497809] 2024-11-25T03:48:36,694 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,33219,1732506497809 already deleted, retry=false 2024-11-25T03:48:36,694 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of d73269e94d72,33219,1732506497809 on d73269e94d72,41803,1732506496940 2024-11-25T03:48:36,699 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure d73269e94d72,33219,1732506497809, splitWal=true, meta=true 2024-11-25T03:48:36,700 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:36,701 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for d73269e94d72,33219,1732506497809 (carryingMeta=true) d73269e94d72,33219,1732506497809/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1cd13f0a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T03:48:36,702 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure d73269e94d72,33219,1732506497809, splitWal=true, meta=true 2024-11-25T03:48:36,704 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure d73269e94d72,33219,1732506497809, splitWal=true, meta=true, isMeta: true 2024-11-25T03:48:36,704 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1] 2024-11-25T03:48:36,706 DEBUG [PEWorker-4 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting 2024-11-25T03:48:36,707 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d73269e94d72:33219 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d73269e94d72/172.17.0.2:33219 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:36,708 INFO [PEWorker-4 {}] master.SplitWALManager(105): d73269e94d72,33219,1732506497809 WAL count=1, meta=true 2024-11-25T03:48:36,708 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1, error=java.net.ConnectException: Call to address=d73269e94d72:33219 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d73269e94d72/172.17.0.2:33219 2024-11-25T03:48:36,708 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-25T03:48:36,708 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 from cache 2024-11-25T03:48:36,709 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address d73269e94d72:33219 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: d73269e94d72/172.17.0.2:33219 2024-11-25T03:48:36,711 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta}] 2024-11-25T03:48:36,719 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=d73269e94d72,38451,1732506497897 2024-11-25T03:48:36,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, worker=d73269e94d72,38451,1732506497897}] 2024-11-25T03:48:36,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:48:36,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33219-0x101701b280d0002, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:48:36,785 INFO [RS:1;d73269e94d72:33219 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:48:36,785 INFO [RS:1;d73269e94d72:33219 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,33219,1732506497809; zookeeper connection closed. 2024-11-25T03:48:36,788 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@11a5f1b9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@11a5f1b9 2024-11-25T03:48:36,885 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T03:48:36,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47447, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T03:48:36,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38451 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-25T03:48:36,914 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, size=7.2 K (7323bytes) 2024-11-25T03:48:36,915 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta 2024-11-25T03:48:36,915 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta after 0ms 2024-11-25T03:48:36,944 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:36,944 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta took 30ms 2024-11-25T03:48:36,952 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-25T03:48:36,953 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta so closing down 2024-11-25T03:48:36,954 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:36,958 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta.temp 2024-11-25T03:48:36,994 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta.temp 2024-11-25T03:48:36,994 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:37,010 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:37,011 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1] 2024-11-25T03:48:37,012 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to d73269e94d72:33219 this server is in the failed servers list 2024-11-25T03:48:37,012 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=d73269e94d72:33219 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d73269e94d72:33219 2024-11-25T03:48:37,012 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d73269e94d72:33219 2024-11-25T03:48:37,012 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 from cache 2024-11-25T03:48:37,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741892_1070 (size=7071) 2024-11-25T03:48:37,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741892_1070 (size=7071) 2024-11-25T03:48:37,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741892_1070 (size=7071) 2024-11-25T03:48:37,022 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:37,023 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta.temp to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T03:48:37,024 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 78 ms; skipped=1; WAL=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false 2024-11-25T03:48:37,024 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, journal: Splitting hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, size=7.2 K (7323bytes) at 1732506516915Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta so closing down at 1732506516953 (+38 ms)Creating recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta.temp at 1732506516994 (+41 ms)3 split writer threads finished at 1732506516994Closed recovered edits writer path=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732506517022 (+28 ms)Rename recovered edits hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta.temp to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1732506517023 (+1 ms)Processed 16 edits across 1 Regions in 78 ms; skipped=1; WAL=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, size=7.2 K, length=7323, corrupted=false, cancelled=false at 1732506517024 (+1 ms) 2024-11-25T03:48:37,024 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta 2024-11-25T03:48:37,025 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-25T03:48:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-25T03:48:37,036 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs 2024-11-25T03:48:37,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-25T03:48:37,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, worker=d73269e94d72,38451,1732506497897 in 315 msec 2024-11-25T03:48:37,044 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=d73269e94d72,38451,1732506497897 2024-11-25T03:48:37,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-25T03:48:37,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C33219%2C1732506497809.meta.1732506500061.meta, worker=d73269e94d72,38451,1732506497897 in 335 msec 2024-11-25T03:48:37,053 INFO [PEWorker-4 {}] master.SplitLogManager(171): hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting dir is empty, no logs to split. 2024-11-25T03:48:37,053 INFO [PEWorker-4 {}] master.SplitWALManager(105): d73269e94d72,33219,1732506497809 WAL count=0, meta=true 2024-11-25T03:48:37,053 DEBUG [PEWorker-4 {}] procedure.ServerCrashProcedure(329): Check if d73269e94d72,33219,1732506497809 WAL splitting is done? wals=0, meta=true 2024-11-25T03:48:37,056 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T03:48:37,058 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:48:37,059 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:48:37,210 DEBUG [d73269e94d72:41803 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T03:48:37,210 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:48:37,211 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:48:37,211 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:48:37,211 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:48:37,211 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:48:37,211 INFO [d73269e94d72:41803 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:48:37,211 INFO [d73269e94d72:41803 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:48:37,211 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:48:37,211 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=d73269e94d72,38451,1732506497897 2024-11-25T03:48:37,213 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,38451,1732506497897, state=OPENING 2024-11-25T03:48:37,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:37,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:37,263 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:37,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:37,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:37,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:37,263 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T03:48:37,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,38451,1732506497897}] 2024-11-25T03:48:37,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-25T03:48:37,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:37,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-25T03:48:37,296 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:37,297 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:48:37,297 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:48:37,424 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T03:48:37,425 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:37,425 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T03:48:37,426 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=d73269e94d72%2C38451%2C1732506497897.meta, suffix=.meta, logDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,38451,1732506497897, archiveDir=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs, maxLogs=32 2024-11-25T03:48:37,439 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,38451,1732506497897/d73269e94d72%2C38451%2C1732506497897.meta.1732506517427.meta, exclude list is [], retry=0 2024-11-25T03:48:37,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:37,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:37,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:37,479 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,38451,1732506497897/d73269e94d72%2C38451%2C1732506497897.meta.1732506517427.meta 2024-11-25T03:48:37,479 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:37,479 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:37,480 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T03:48:37,480 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T03:48:37,480 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T03:48:37,480 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T03:48:37,480 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:37,481 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T03:48:37,481 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T03:48:37,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-25T03:48:37,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T03:48:37,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:37,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:37,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-25T03:48:37,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T03:48:37,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:37,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:37,488 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-25T03:48:37,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T03:48:37,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:37,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:37,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-25T03:48:37,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T03:48:37,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:37,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T03:48:37,491 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T03:48:37,492 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740 2024-11-25T03:48:37,495 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740 2024-11-25T03:48:37,496 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T03:48:37,519 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:37,522 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,38451,1732506497897, seqNum=-1] 2024-11-25T03:48:37,523 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:48:37,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:48:37,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38451 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Get size: 119 connection: 172.17.0.2:50466 deadline: 1732506577526, exception=org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on d73269e94d72,38451,1732506497897 2024-11-25T03:48:37,528 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,38451,1732506497897, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d73269e94d72,38451,1732506497897, seqNum=-1, error=org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on d73269e94d72,38451,1732506497897 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:48:37,528 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,38451,1732506497897, seqNum=-1 is org.apache.hadoop.hbase.exceptions.RegionOpeningException: org.apache.hadoop.hbase.exceptions.RegionOpeningException: Region hbase:meta,,1 is opening on d73269e94d72,38451,1732506497897 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3183) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.get(RSRpcServices.java:2474) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43504) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-25T03:48:37,528 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d73269e94d72,38451,1732506497897, seqNum=-1 from cache 2024-11-25T03:48:37,535 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:37,538 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T03:48:37,538 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-25T03:48:37,567 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/info/59162c568842440192cbee68db038e31 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462./info:regioninfo/1732506516287/Put/seqid=0 2024-11-25T03:48:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741894_1072 (size=11177) 2024-11-25T03:48:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741894_1072 (size=11177) 2024-11-25T03:48:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741894_1072 (size=11177) 2024-11-25T03:48:37,580 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/info/59162c568842440192cbee68db038e31 2024-11-25T03:48:37,609 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/ns/10defdf1c13c4d6794ebac061122e6f5 is 43, key is default/ns:d/1732506500285/Put/seqid=0 2024-11-25T03:48:37,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741895_1073 (size=5153) 2024-11-25T03:48:37,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741895_1073 (size=5153) 2024-11-25T03:48:37,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741895_1073 (size=5153) 2024-11-25T03:48:37,624 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/ns/10defdf1c13c4d6794ebac061122e6f5 2024-11-25T03:48:37,650 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/table/e33f351452bd45c3b068a62020d35ede is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732506514190/Put/seqid=0 2024-11-25T03:48:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741896_1074 (size=5431) 2024-11-25T03:48:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741896_1074 (size=5431) 2024-11-25T03:48:37,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741896_1074 (size=5431) 2024-11-25T03:48:37,657 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/table/e33f351452bd45c3b068a62020d35ede 2024-11-25T03:48:37,670 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/info/59162c568842440192cbee68db038e31 as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/info/59162c568842440192cbee68db038e31 2024-11-25T03:48:37,678 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/info/59162c568842440192cbee68db038e31, entries=36, sequenceid=18, filesize=10.9 K 2024-11-25T03:48:37,679 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/ns/10defdf1c13c4d6794ebac061122e6f5 as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/ns/10defdf1c13c4d6794ebac061122e6f5 2024-11-25T03:48:37,686 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/ns/10defdf1c13c4d6794ebac061122e6f5, entries=2, sequenceid=18, filesize=5.0 K 2024-11-25T03:48:37,687 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/table/e33f351452bd45c3b068a62020d35ede as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/table/e33f351452bd45c3b068a62020d35ede 2024-11-25T03:48:37,697 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/table/e33f351452bd45c3b068a62020d35ede, entries=2, sequenceid=18, filesize=5.3 K 2024-11-25T03:48:37,697 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 159ms, sequenceid=18, compaction requested=false; wal=null 2024-11-25T03:48:37,697 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T03:48:37,698 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-25T03:48:37,700 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T03:48:37,700 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T03:48:37,701 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T03:48:37,703 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T03:48:37,705 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-25T03:48:37,706 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66034036, jitterRate=-0.01601618528366089}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T03:48:37,706 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T03:48:37,707 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732506517481Writing region info on filesystem at 1732506517481Initializing all the Stores at 1732506517482 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506517482Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506517483 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506517483Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732506517484 (+1 ms)Obtaining lock to block concurrent updates at 1732506517538 (+54 ms)Preparing flush snapshotting stores in 1588230740 at 1732506517538Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1732506517539 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732506517539Flushing 1588230740/info: creating writer at 1732506517539Flushing 1588230740/info: appending metadata at 1732506517566 (+27 ms)Flushing 1588230740/info: closing flushed file at 1732506517566Flushing 1588230740/ns: creating writer at 1732506517588 (+22 ms)Flushing 1588230740/ns: appending metadata at 1732506517609 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732506517609Flushing 1588230740/table: creating writer at 1732506517630 (+21 ms)Flushing 1588230740/table: appending metadata at 1732506517649 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732506517649Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b2e05a8: reopening flushed file at 1732506517668 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f78fcc9: reopening flushed file at 1732506517678 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@130af14c: reopening flushed file at 1732506517686 (+8 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 159ms, sequenceid=18, compaction requested=false; wal=null at 1732506517697 (+11 ms)Cleaning up temporary data from old regions at 1732506517700 (+3 ms)Running coprocessor post-open hooks at 1732506517706 (+6 ms)Region opened successfully at 1732506517706 2024-11-25T03:48:37,708 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1732506517421 2024-11-25T03:48:37,710 DEBUG [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T03:48:37,710 INFO [RS_OPEN_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T03:48:37,712 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=d73269e94d72,38451,1732506497897 2024-11-25T03:48:37,714 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as d73269e94d72,38451,1732506497897, state=OPEN 2024-11-25T03:48:37,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:37,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:37,768 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:37,768 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d73269e94d72,38451,1732506497897, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:48:37,768 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:37,768 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=d73269e94d72,38451,1732506497897 2024-11-25T03:48:37,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T03:48:37,768 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T03:48:37,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-25T03:48:37,772 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=d73269e94d72,38451,1732506497897 in 505 msec 2024-11-25T03:48:37,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-25T03:48:37,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 717 msec 2024-11-25T03:48:37,776 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): d73269e94d72,33219,1732506497809 had 2 regions 2024-11-25T03:48:37,777 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure d73269e94d72,33219,1732506497809, splitWal=true, meta=true, isMeta: false 2024-11-25T03:48:37,779 INFO [PEWorker-2 {}] master.SplitWALManager(105): d73269e94d72,33219,1732506497809 WAL count=1, meta=false 2024-11-25T03:48:37,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure d73269e94d72%2C33219%2C1732506497809.1732506499467}] 2024-11-25T03:48:37,782 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=d73269e94d72,38451,1732506497897 2024-11-25T03:48:37,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C33219%2C1732506497809.1732506499467, worker=d73269e94d72,38451,1732506497897}] 2024-11-25T03:48:37,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38451 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-25T03:48:37,959 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467, size=2.2 K (2288bytes) 2024-11-25T03:48:37,960 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467 2024-11-25T03:48:37,960 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467 after 0ms 2024-11-25T03:48:37,988 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:37,988 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467 took 29ms 2024-11-25T03:48:37,992 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 1b485cca92b597d29ec36b9eadec9462: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-25T03:48:37,992 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467 so closing down 2024-11-25T03:48:37,992 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:37,992 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:37,992 INFO [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-25T03:48:37,993 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467, journal: Splitting hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467, size=2.2 K (2288bytes) at 1732506517960Finishing writing output for hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467 so closing down at 1732506517992 (+32 ms)3 split writer threads finished at 1732506517992Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732506517992 2024-11-25T03:48:37,993 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467 2024-11-25T03:48:37,993 DEBUG [RS_LOG_REPLAY_OPS-regionserver/d73269e94d72:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-25T03:48:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41803 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-25T03:48:37,999 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting/d73269e94d72%2C33219%2C1732506497809.1732506499467 to hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs 2024-11-25T03:48:38,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-25T03:48:38,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure d73269e94d72%2C33219%2C1732506497809.1732506499467, worker=d73269e94d72,38451,1732506497897 in 216 msec 2024-11-25T03:48:38,003 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=d73269e94d72,38451,1732506497897 2024-11-25T03:48:38,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-25T03:48:38,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure d73269e94d72%2C33219%2C1732506497809.1732506499467, worker=d73269e94d72,38451,1732506497897 in 225 msec 2024-11-25T03:48:38,008 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/WALs/d73269e94d72,33219,1732506497809-splitting dir is empty, no logs to split. 2024-11-25T03:48:38,008 INFO [PEWorker-2 {}] master.SplitWALManager(105): d73269e94d72,33219,1732506497809 WAL count=0, meta=false 2024-11-25T03:48:38,008 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if d73269e94d72,33219,1732506497809 WAL splitting is done? wals=0, meta=false 2024-11-25T03:48:38,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN}] 2024-11-25T03:48:38,012 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN 2024-11-25T03:48:38,013 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T03:48:38,163 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(204): Hosts are {d73269e94d72=0} racks are {/default-rack=0} 2024-11-25T03:48:38,164 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T03:48:38,164 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T03:48:38,164 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T03:48:38,164 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T03:48:38,164 INFO [d73269e94d72:41803 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T03:48:38,164 INFO [d73269e94d72:41803 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T03:48:38,164 DEBUG [d73269e94d72:41803 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T03:48:38,164 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPENING, regionLocation=d73269e94d72,33171,1732506497711 2024-11-25T03:48:38,165 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server d73269e94d72:33219 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d73269e94d72/172.17.0.2:33219 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:38,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1, error=java.net.ConnectException: Call to address=d73269e94d72:33219 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d73269e94d72/172.17.0.2:33219 2024-11-25T03:48:38,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-25T03:48:38,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=d73269e94d72,33219,1732506497809, seqNum=-1 from cache 2024-11-25T03:48:38,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address d73269e94d72:33219 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: d73269e94d72/172.17.0.2:33219 2024-11-25T03:48:38,270 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:38,270 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,38451,1732506497897, seqNum=-1] 2024-11-25T03:48:38,270 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T03:48:38,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51371, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T03:48:38,273 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN because future has completed 2024-11-25T03:48:38,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711}] 2024-11-25T03:48:38,430 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:38,430 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b485cca92b597d29ec36b9eadec9462, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:38,430 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,430 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:38,431 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,431 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,436 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,437 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf1 2024-11-25T03:48:38,437 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:38,445 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf1/1947dbdd0b6d4b719d1d9c8446b811ee 2024-11-25T03:48:38,445 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:38,445 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,447 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b485cca92b597d29ec36b9eadec9462 columnFamilyName cf2 2024-11-25T03:48:38,447 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:38,455 DEBUG [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/cf2/118c813ea5c443ceb16bb916d2a7598a 2024-11-25T03:48:38,456 INFO [StoreOpener-1b485cca92b597d29ec36b9eadec9462-1 {}] regionserver.HStore(327): Store=1b485cca92b597d29ec36b9eadec9462/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:38,456 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,457 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,460 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,462 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,462 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,463 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-25T03:48:38,464 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,465 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 1b485cca92b597d29ec36b9eadec9462; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58846248, jitterRate=-0.12312257289886475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-25T03:48:38,465 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:48:38,466 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 1b485cca92b597d29ec36b9eadec9462: Running coprocessor pre-open hook at 1732506518431Writing region info on filesystem at 1732506518431Initializing all the Stores at 1732506518432 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506518432Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506518436 (+4 ms)Cleaning up temporary data from old regions at 1732506518462 (+26 ms)Running coprocessor post-open hooks at 1732506518465 (+3 ms)Region opened successfully at 1732506518466 (+1 ms) 2024-11-25T03:48:38,467 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., pid=21, masterSystemTime=1732506518426 2024-11-25T03:48:38,470 DEBUG [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:38,470 INFO [RS_OPEN_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:48:38,471 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1b485cca92b597d29ec36b9eadec9462, regionState=OPEN, openSeqNum=18, regionLocation=d73269e94d72,33171,1732506497711 2024-11-25T03:48:38,474 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41803 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=d73269e94d72,33171,1732506497711, table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T03:48:38,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711 because future has completed 2024-11-25T03:48:38,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-25T03:48:38,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b485cca92b597d29ec36b9eadec9462, server=d73269e94d72,33171,1732506497711 in 205 msec 2024-11-25T03:48:38,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-25T03:48:38,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=1b485cca92b597d29ec36b9eadec9462, ASSIGN in 474 msec 2024-11-25T03:48:38,488 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server d73269e94d72,33219,1732506497809 after splitting done 2024-11-25T03:48:38,498 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure d73269e94d72,33219,1732506497809, splitWal=true, meta=true in 1.7930 sec 2024-11-25T03:48:38,540 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T03:48:38,541 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=d73269e94d72,38451,1732506497897, seqNum=-1] 2024-11-25T03:48:38,670 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to d73269e94d72:33219 this server is in the failed servers list 2024-11-25T03:48:38,670 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=d73269e94d72:33219 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d73269e94d72:33219 2024-11-25T03:48:38,670 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: d73269e94d72:33219 2024-11-25T03:48:38,670 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33219,1732506497809, seqNum=12 from cache 2024-11-25T03:48:38,886 DEBUG [RPCClient-NioEventLoopGroup-6-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462., hostname=d73269e94d72,33171,1732506497711, seqNum=18] 2024-11-25T03:48:38,902 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=407 (was 411), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 411) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8619 (was 8672) 2024-11-25T03:48:38,904 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-25T03:48:38,920 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=407, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=8615 2024-11-25T03:48:38,920 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-11-25T03:48:38,940 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:38,942 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:38,943 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:38,946 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-49167778, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-49167778, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:38,960 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-49167778/hregion-49167778.1732506518946, exclude list is [], retry=0 2024-11-25T03:48:38,963 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:38,963 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:38,963 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:38,991 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-49167778/hregion-49167778.1732506518946 2024-11-25T03:48:38,992 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:38,992 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a747743145bee7b04de735296e880383, NAME => 'testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:39,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741898_1076 (size=67) 2024-11-25T03:48:39,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741898_1076 (size=67) 2024-11-25T03:48:39,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741898_1076 (size=67) 2024-11-25T03:48:39,014 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:39,015 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,017 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName a 2024-11-25T03:48:39,017 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,018 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,018 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,019 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName b 2024-11-25T03:48:39,020 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,020 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,020 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,022 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName c 2024-11-25T03:48:39,022 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,022 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,023 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383 2024-11-25T03:48:39,024 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383 2024-11-25T03:48:39,024 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,024 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,025 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:39,026 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,028 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:39,029 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a747743145bee7b04de735296e880383; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67568120, jitterRate=0.006843447685241699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:39,030 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a747743145bee7b04de735296e880383: Writing region info on filesystem at 1732506519014Initializing all the Stores at 1732506519015 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519015Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519015Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519015Cleaning up temporary data from old regions at 1732506519024 (+9 ms)Region opened successfully at 1732506519029 (+5 ms) 2024-11-25T03:48:39,030 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a747743145bee7b04de735296e880383, disabling compactions & flushes 2024-11-25T03:48:39,030 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,030 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,030 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. after waiting 0 ms 2024-11-25T03:48:39,030 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,030 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,030 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a747743145bee7b04de735296e880383: Waiting for close lock at 1732506519030Disabling compacts and flushes for region at 1732506519030Disabling writes for close at 1732506519030Writing region close event to WAL at 1732506519030Closed at 1732506519030 2024-11-25T03:48:39,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741897_1075 (size=175) 2024-11-25T03:48:39,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741897_1075 (size=175) 2024-11-25T03:48:39,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741897_1075 (size=175) 2024-11-25T03:48:39,036 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:39,036 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-49167778:(num 1732506518946) 2024-11-25T03:48:39,036 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:39,038 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:39,051 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038, exclude list is [], retry=0 2024-11-25T03:48:39,054 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:39,054 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:39,055 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:39,084 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038 2024-11-25T03:48:39,085 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:39,085 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a747743145bee7b04de735296e880383, NAME => 'testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:39,085 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:39,085 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,085 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,087 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,088 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName a 2024-11-25T03:48:39,088 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,088 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,088 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,089 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName b 2024-11-25T03:48:39,089 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,090 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,090 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,091 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName c 2024-11-25T03:48:39,091 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,091 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,091 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,092 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383 2024-11-25T03:48:39,093 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383 2024-11-25T03:48:39,094 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,094 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,094 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:39,095 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,096 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a747743145bee7b04de735296e880383; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61806285, jitterRate=-0.07901458442211151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:39,096 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a747743145bee7b04de735296e880383: Writing region info on filesystem at 1732506519086Initializing all the Stores at 1732506519086Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519086Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519087 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519087Cleaning up temporary data from old regions at 1732506519094 (+7 ms)Region opened successfully at 1732506519096 (+2 ms) 2024-11-25T03:48:39,137 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a747743145bee7b04de735296e880383 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T03:48:39,176 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/a/ef3ee33c3bb84275aa36fb222961a281 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732506519096/Put/seqid=0 2024-11-25T03:48:39,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741900_1078 (size=5958) 2024-11-25T03:48:39,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741900_1078 (size=5958) 2024-11-25T03:48:39,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741900_1078 (size=5958) 2024-11-25T03:48:39,228 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/a/ef3ee33c3bb84275aa36fb222961a281 2024-11-25T03:48:39,263 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/b/05a09b9639fe4d14a4d3607eb31b74b6 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732506519113/Put/seqid=0 2024-11-25T03:48:39,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741901_1079 (size=5958) 2024-11-25T03:48:39,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741901_1079 (size=5958) 2024-11-25T03:48:39,291 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/b/05a09b9639fe4d14a4d3607eb31b74b6 2024-11-25T03:48:39,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741901_1079 (size=5958) 2024-11-25T03:48:39,326 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/c/534358cc85cf4078b115ed82727e4141 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732506519122/Put/seqid=0 2024-11-25T03:48:39,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741902_1080 (size=5958) 2024-11-25T03:48:39,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741902_1080 (size=5958) 2024-11-25T03:48:39,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741902_1080 (size=5958) 2024-11-25T03:48:39,363 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/c/534358cc85cf4078b115ed82727e4141 2024-11-25T03:48:39,372 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/a/ef3ee33c3bb84275aa36fb222961a281 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/a/ef3ee33c3bb84275aa36fb222961a281 2024-11-25T03:48:39,384 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/a/ef3ee33c3bb84275aa36fb222961a281, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T03:48:39,387 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/b/05a09b9639fe4d14a4d3607eb31b74b6 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/b/05a09b9639fe4d14a4d3607eb31b74b6 2024-11-25T03:48:39,394 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/b/05a09b9639fe4d14a4d3607eb31b74b6, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T03:48:39,405 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/c/534358cc85cf4078b115ed82727e4141 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/c/534358cc85cf4078b115ed82727e4141 2024-11-25T03:48:39,414 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/c/534358cc85cf4078b115ed82727e4141, entries=10, sequenceid=33, filesize=5.8 K 2024-11-25T03:48:39,416 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for a747743145bee7b04de735296e880383 in 279ms, sequenceid=33, compaction requested=false 2024-11-25T03:48:39,416 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a747743145bee7b04de735296e880383: 2024-11-25T03:48:39,416 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a747743145bee7b04de735296e880383, disabling compactions & flushes 2024-11-25T03:48:39,416 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,416 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,416 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. after waiting 0 ms 2024-11-25T03:48:39,416 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,430 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. 2024-11-25T03:48:39,430 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a747743145bee7b04de735296e880383: Waiting for close lock at 1732506519416Disabling compacts and flushes for region at 1732506519416Disabling writes for close at 1732506519416Writing region close event to WAL at 1732506519430 (+14 ms)Closed at 1732506519430 2024-11-25T03:48:39,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741899_1077 (size=6311) 2024-11-25T03:48:39,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741899_1077 (size=6311) 2024-11-25T03:48:39,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741899_1077 (size=6311) 2024-11-25T03:48:39,450 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/b/05a09b9639fe4d14a4d3607eb31b74b6 to hdfs://localhost:46839/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/b/05a09b9639fe4d14a4d3607eb31b74b6 2024-11-25T03:48:39,472 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038, size=6.2 K (6311bytes) 2024-11-25T03:48:39,472 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038 2024-11-25T03:48:39,473 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038 after 1ms 2024-11-25T03:48:39,508 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:39,508 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038 took 36ms 2024-11-25T03:48:39,511 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038 so closing down 2024-11-25T03:48:39,511 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:39,515 INFO [Time-limited test-Writer-1 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732506519038.temp 2024-11-25T03:48:39,561 INFO [Time-limited test-Writer-1 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000003-wal.1732506519038.temp 2024-11-25T03:48:39,561 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:39,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741903_1081 (size=5575) 2024-11-25T03:48:39,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741903_1081 (size=5575) 2024-11-25T03:48:39,585 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000003-wal.1732506519038.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:39,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741903_1081 (size=5575) 2024-11-25T03:48:39,588 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000003-wal.1732506519038.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000032 2024-11-25T03:48:39,588 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 79 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-25T03:48:39,588 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038, size=6.2 K (6311bytes) at 1732506519472Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038 so closing down at 1732506519511 (+39 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000003-wal.1732506519038.temp at 1732506519561 (+50 ms)3 split writer threads finished at 1732506519561Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000003-wal.1732506519038.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732506519585 (+24 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000003-wal.1732506519038.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000032 at 1732506519588 (+3 ms)Processed 32 edits across 1 Regions in 79 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732506519588 2024-11-25T03:48:39,590 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519038 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506519038 2024-11-25T03:48:39,591 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000032 2024-11-25T03:48:39,591 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:39,593 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:39,624 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519593, exclude list is [], retry=0 2024-11-25T03:48:39,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:39,627 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:39,628 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:39,678 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519593 2024-11-25T03:48:39,678 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:39,678 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a747743145bee7b04de735296e880383, NAME => 'testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:39,679 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:39,679 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,679 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,681 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,683 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName a 2024-11-25T03:48:39,683 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,690 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:48:39,699 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/a/ef3ee33c3bb84275aa36fb222961a281 2024-11-25T03:48:39,699 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,700 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,716 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName b 2024-11-25T03:48:39,717 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,719 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,720 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a747743145bee7b04de735296e880383 2024-11-25T03:48:39,721 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a747743145bee7b04de735296e880383 columnFamilyName c 2024-11-25T03:48:39,721 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:39,751 DEBUG [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/c/534358cc85cf4078b115ed82727e4141 2024-11-25T03:48:39,751 INFO [StoreOpener-a747743145bee7b04de735296e880383-1 {}] regionserver.HStore(327): Store=a747743145bee7b04de735296e880383/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:39,751 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,754 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383 2024-11-25T03:48:39,757 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383 2024-11-25T03:48:39,758 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000032 2024-11-25T03:48:39,801 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:39,803 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000032 2024-11-25T03:48:39,804 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a747743145bee7b04de735296e880383 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T03:48:39,835 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/b/4464a7baa98d45419bcb5f2268d4f8d4 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732506519113/Put/seqid=0 2024-11-25T03:48:39,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741905_1083 (size=5958) 2024-11-25T03:48:39,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741905_1083 (size=5958) 2024-11-25T03:48:39,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741905_1083 (size=5958) 2024-11-25T03:48:39,864 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/b/4464a7baa98d45419bcb5f2268d4f8d4 2024-11-25T03:48:39,875 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/.tmp/b/4464a7baa98d45419bcb5f2268d4f8d4 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/b/4464a7baa98d45419bcb5f2268d4f8d4 2024-11-25T03:48:39,887 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/b/4464a7baa98d45419bcb5f2268d4f8d4, entries=10, sequenceid=32, filesize=5.8 K 2024-11-25T03:48:39,887 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a747743145bee7b04de735296e880383 in 84ms, sequenceid=32, compaction requested=false; wal=null 2024-11-25T03:48:39,889 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/0000000000000000032 2024-11-25T03:48:39,892 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,893 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,893 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:39,896 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a747743145bee7b04de735296e880383 2024-11-25T03:48:39,899 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/a747743145bee7b04de735296e880383/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-25T03:48:39,902 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a747743145bee7b04de735296e880383; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66600972, jitterRate=-0.007568180561065674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:39,903 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a747743145bee7b04de735296e880383: Writing region info on filesystem at 1732506519679Initializing all the Stores at 1732506519680 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519680Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519681 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506519681Obtaining lock to block concurrent updates at 1732506519804 (+123 ms)Preparing flush snapshotting stores in a747743145bee7b04de735296e880383 at 1732506519804Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732506519804Flushing stores of testReplayEditsWrittenViaHRegion,,1732506518941.a747743145bee7b04de735296e880383. at 1732506519804Flushing a747743145bee7b04de735296e880383/b: creating writer at 1732506519804Flushing a747743145bee7b04de735296e880383/b: appending metadata at 1732506519835 (+31 ms)Flushing a747743145bee7b04de735296e880383/b: closing flushed file at 1732506519835Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77f21c3e: reopening flushed file at 1732506519874 (+39 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for a747743145bee7b04de735296e880383 in 84ms, sequenceid=32, compaction requested=false; wal=null at 1732506519887 (+13 ms)Cleaning up temporary data from old regions at 1732506519893 (+6 ms)Region opened successfully at 1732506519902 (+9 ms) 2024-11-25T03:48:39,954 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=415 (was 407) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:48316 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:39984 [Waiting for operation #51] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:48546 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:55704 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:40142 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:55890 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1116 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=418 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=8585 (was 8615) 2024-11-25T03:48:39,955 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-25T03:48:39,970 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1116, MaxFileDescriptor=1048576, SystemLoadAverage=418, ProcessCount=11, AvailableMemoryMB=8584 2024-11-25T03:48:39,970 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1116 is superior to 1024 2024-11-25T03:48:39,989 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:39,992 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:39,993 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:39,996 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92054288, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-92054288, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:40,014 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92054288/hregion-92054288.1732506519997, exclude list is [], retry=0 2024-11-25T03:48:40,017 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:40,017 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:40,017 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:40,053 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92054288/hregion-92054288.1732506519997 2024-11-25T03:48:40,053 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:40,054 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 874e3c13b978726b19d61a2442630d8e, NAME => 'testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:40,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741907_1085 (size=68) 2024-11-25T03:48:40,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741907_1085 (size=68) 2024-11-25T03:48:40,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741907_1085 (size=68) 2024-11-25T03:48:40,100 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:40,101 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,103 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName a 2024-11-25T03:48:40,103 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:40,103 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:40,104 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,105 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName b 2024-11-25T03:48:40,105 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:40,106 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:40,106 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,107 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName c 2024-11-25T03:48:40,107 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:40,108 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:40,108 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,109 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,110 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,110 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,111 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,111 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:40,114 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,116 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:40,116 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 874e3c13b978726b19d61a2442630d8e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60420297, jitterRate=-0.09966741502285004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:40,117 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 874e3c13b978726b19d61a2442630d8e: Writing region info on filesystem at 1732506520100Initializing all the Stores at 1732506520101 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506520101Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506520101Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506520101Cleaning up temporary data from old regions at 1732506520111 (+10 ms)Region opened successfully at 1732506520117 (+6 ms) 2024-11-25T03:48:40,117 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 874e3c13b978726b19d61a2442630d8e, disabling compactions & flushes 2024-11-25T03:48:40,117 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:40,117 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:40,118 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. after waiting 0 ms 2024-11-25T03:48:40,118 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:40,118 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:40,118 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 874e3c13b978726b19d61a2442630d8e: Waiting for close lock at 1732506520117Disabling compacts and flushes for region at 1732506520117Disabling writes for close at 1732506520118 (+1 ms)Writing region close event to WAL at 1732506520118Closed at 1732506520118 2024-11-25T03:48:40,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741906_1084 (size=175) 2024-11-25T03:48:40,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741906_1084 (size=175) 2024-11-25T03:48:40,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741906_1084 (size=175) 2024-11-25T03:48:40,126 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:40,126 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92054288:(num 1732506519997) 2024-11-25T03:48:40,127 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:40,128 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:40,142 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129, exclude list is [], retry=0 2024-11-25T03:48:40,145 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:40,145 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:40,145 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:40,182 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129 2024-11-25T03:48:40,182 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:40,257 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 874e3c13b978726b19d61a2442630d8e, NAME => 'testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:40,259 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,259 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:40,259 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,259 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,261 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,262 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName a 2024-11-25T03:48:40,262 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:40,262 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:40,263 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,264 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName b 2024-11-25T03:48:40,264 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:40,264 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:40,265 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,265 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName c 2024-11-25T03:48:40,266 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:40,266 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:40,266 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,272 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,272 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,273 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:40,274 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,275 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 874e3c13b978726b19d61a2442630d8e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71315254, jitterRate=0.06268009543418884}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:40,275 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:40,276 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 874e3c13b978726b19d61a2442630d8e: Running coprocessor pre-open hook at 1732506520259Writing region info on filesystem at 1732506520259Initializing all the Stores at 1732506520260 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506520260Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506520260Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506520260Cleaning up temporary data from old regions at 1732506520272 (+12 ms)Running coprocessor post-open hooks at 1732506520275 (+3 ms)Region opened successfully at 1732506520276 (+1 ms) 2024-11-25T03:48:40,291 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 874e3c13b978726b19d61a2442630d8e 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-25T03:48:40,294 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:41,295 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:42,296 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:43,297 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:44,298 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:45,299 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:45,762 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:48:46,031 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-25T03:48:46,299 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:47,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-25T03:48:47,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:47,293 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T03:48:47,293 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:47,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-25T03:48:47,294 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:47,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-25T03:48:47,295 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:47,295 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:48:47,295 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T03:48:47,300 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:48,301 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:49,302 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 874e3c13b978726b19d61a2442630d8e/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:49,303 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 874e3c13b978726b19d61a2442630d8e: 2024-11-25T03:48:49,303 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:49,319 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 874e3c13b978726b19d61a2442630d8e: 2024-11-25T03:48:49,319 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-25T03:48:49,319 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 874e3c13b978726b19d61a2442630d8e, disabling compactions & flushes 2024-11-25T03:48:49,319 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:49,320 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:49,320 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. after waiting 0 ms 2024-11-25T03:48:49,320 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:49,320 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:49,320 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. 2024-11-25T03:48:49,320 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 874e3c13b978726b19d61a2442630d8e: Waiting for close lock at 1732506529319Running coprocessor pre-close hooks at 1732506529319Disabling compacts and flushes for region at 1732506529319Disabling writes for close at 1732506529320 (+1 ms)Writing region close event to WAL at 1732506529320Running coprocessor post-close hooks at 1732506529320Closed at 1732506529320 2024-11-25T03:48:49,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741908_1086 (size=4236) 2024-11-25T03:48:49,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741908_1086 (size=4236) 2024-11-25T03:48:49,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741908_1086 (size=4236) 2024-11-25T03:48:49,356 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129, size=4.1 K (4236bytes) 2024-11-25T03:48:49,356 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129 2024-11-25T03:48:49,356 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129 after 0ms 2024-11-25T03:48:49,387 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:49,387 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129 took 31ms 2024-11-25T03:48:49,389 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129 so closing down 2024-11-25T03:48:49,389 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:49,390 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732506520129.temp 2024-11-25T03:48:49,421 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000004-wal.1732506520129.temp 2024-11-25T03:48:49,421 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:49,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741909_1087 (size=3245) 2024-11-25T03:48:49,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741909_1087 (size=3245) 2024-11-25T03:48:49,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741909_1087 (size=3245) 2024-11-25T03:48:49,435 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000004-wal.1732506520129.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:49,436 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000004-wal.1732506520129.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000026 2024-11-25T03:48:49,436 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 49 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-25T03:48:49,436 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129, size=4.1 K (4236bytes) at 1732506529356Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129 so closing down at 1732506529389 (+33 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000004-wal.1732506520129.temp at 1732506529421 (+32 ms)3 split writer threads finished at 1732506529421Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000004-wal.1732506520129.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732506529435 (+14 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000004-wal.1732506520129.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000026 at 1732506529436 (+1 ms)Processed 23 edits across 1 Regions in 49 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732506529436 2024-11-25T03:48:49,440 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506520129 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506520129 2024-11-25T03:48:49,441 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000026 2024-11-25T03:48:49,441 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:49,443 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:49,461 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506529443, exclude list is [], retry=0 2024-11-25T03:48:49,463 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:49,464 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:49,464 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:49,540 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506529443 2024-11-25T03:48:49,544 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:49,545 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 874e3c13b978726b19d61a2442630d8e, NAME => 'testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:49,545 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,546 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:49,546 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,546 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,549 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,550 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName a 2024-11-25T03:48:49,550 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:49,550 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:49,551 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,551 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName b 2024-11-25T03:48:49,551 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:49,552 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:49,552 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,553 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 874e3c13b978726b19d61a2442630d8e columnFamilyName c 2024-11-25T03:48:49,553 DEBUG [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:49,553 INFO [StoreOpener-874e3c13b978726b19d61a2442630d8e-1 {}] regionserver.HStore(327): Store=874e3c13b978726b19d61a2442630d8e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:49,553 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,554 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,557 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000026 2024-11-25T03:48:49,599 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:49,609 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000026 2024-11-25T03:48:49,609 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 874e3c13b978726b19d61a2442630d8e 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-25T03:48:49,633 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/a/5c47d959cb2b466580368c1e6d43ba39 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732506529308/Put/seqid=0 2024-11-25T03:48:49,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741911_1089 (size=5523) 2024-11-25T03:48:49,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741911_1089 (size=5523) 2024-11-25T03:48:49,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741911_1089 (size=5523) 2024-11-25T03:48:49,647 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/a/5c47d959cb2b466580368c1e6d43ba39 2024-11-25T03:48:49,679 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/b/09543d8773fa4bfc8f71257d0ba72ad4 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732506529304/Put/seqid=0 2024-11-25T03:48:49,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741912_1090 (size=5524) 2024-11-25T03:48:49,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741912_1090 (size=5524) 2024-11-25T03:48:49,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741912_1090 (size=5524) 2024-11-25T03:48:49,692 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/b/09543d8773fa4bfc8f71257d0ba72ad4 2024-11-25T03:48:49,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/c/fa80b5f2ec5f470099304fc1fc39ba9e is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732506529305/Put/seqid=0 2024-11-25T03:48:49,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741913_1091 (size=5457) 2024-11-25T03:48:49,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741913_1091 (size=5457) 2024-11-25T03:48:49,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741913_1091 (size=5457) 2024-11-25T03:48:49,731 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/c/fa80b5f2ec5f470099304fc1fc39ba9e 2024-11-25T03:48:49,741 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/a/5c47d959cb2b466580368c1e6d43ba39 as hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/a/5c47d959cb2b466580368c1e6d43ba39 2024-11-25T03:48:49,747 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/a/5c47d959cb2b466580368c1e6d43ba39, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T03:48:49,748 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/b/09543d8773fa4bfc8f71257d0ba72ad4 as hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/b/09543d8773fa4bfc8f71257d0ba72ad4 2024-11-25T03:48:49,759 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/b/09543d8773fa4bfc8f71257d0ba72ad4, entries=7, sequenceid=26, filesize=5.4 K 2024-11-25T03:48:49,760 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/.tmp/c/fa80b5f2ec5f470099304fc1fc39ba9e as hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/c/fa80b5f2ec5f470099304fc1fc39ba9e 2024-11-25T03:48:49,769 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/c/fa80b5f2ec5f470099304fc1fc39ba9e, entries=6, sequenceid=26, filesize=5.3 K 2024-11-25T03:48:49,769 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 874e3c13b978726b19d61a2442630d8e in 160ms, sequenceid=26, compaction requested=false; wal=null 2024-11-25T03:48:49,771 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/0000000000000000026 2024-11-25T03:48:49,772 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,772 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,773 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:49,774 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,777 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsAfterAbortingFlush/874e3c13b978726b19d61a2442630d8e/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-25T03:48:49,778 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 874e3c13b978726b19d61a2442630d8e; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74125994, jitterRate=0.1045633852481842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:49,778 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 874e3c13b978726b19d61a2442630d8e 2024-11-25T03:48:49,779 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 874e3c13b978726b19d61a2442630d8e: Running coprocessor pre-open hook at 1732506529546Writing region info on filesystem at 1732506529546Initializing all the Stores at 1732506529547 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506529547Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506529548 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506529548Obtaining lock to block concurrent updates at 1732506529609 (+61 ms)Preparing flush snapshotting stores in 874e3c13b978726b19d61a2442630d8e at 1732506529609Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732506529617 (+8 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1732506519991.874e3c13b978726b19d61a2442630d8e. at 1732506529617Flushing 874e3c13b978726b19d61a2442630d8e/a: creating writer at 1732506529617Flushing 874e3c13b978726b19d61a2442630d8e/a: appending metadata at 1732506529633 (+16 ms)Flushing 874e3c13b978726b19d61a2442630d8e/a: closing flushed file at 1732506529633Flushing 874e3c13b978726b19d61a2442630d8e/b: creating writer at 1732506529662 (+29 ms)Flushing 874e3c13b978726b19d61a2442630d8e/b: appending metadata at 1732506529679 (+17 ms)Flushing 874e3c13b978726b19d61a2442630d8e/b: closing flushed file at 1732506529679Flushing 874e3c13b978726b19d61a2442630d8e/c: creating writer at 1732506529698 (+19 ms)Flushing 874e3c13b978726b19d61a2442630d8e/c: appending metadata at 1732506529713 (+15 ms)Flushing 874e3c13b978726b19d61a2442630d8e/c: closing flushed file at 1732506529713Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fcd6379: reopening flushed file at 1732506529737 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5dc360af: reopening flushed file at 1732506529747 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34b42f24: reopening flushed file at 1732506529759 (+12 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 874e3c13b978726b19d61a2442630d8e in 160ms, sequenceid=26, compaction requested=false; wal=null at 1732506529769 (+10 ms)Cleaning up temporary data from old regions at 1732506529772 (+3 ms)Running coprocessor post-open hooks at 1732506529778 (+6 ms)Region opened successfully at 1732506529779 (+1 ms) 2024-11-25T03:48:49,801 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 415) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:54822 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:39098 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:53960 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:54810 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:39106 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:53946 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1116) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 418), ProcessCount=11 (was 11), AvailableMemoryMB=8571 (was 8584) 2024-11-25T03:48:49,801 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-25T03:48:49,816 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=8571 2024-11-25T03:48:49,816 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-25T03:48:49,833 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:49,835 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:49,836 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:49,838 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45209888, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-45209888, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:49,852 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45209888/hregion-45209888.1732506529838, exclude list is [], retry=0 2024-11-25T03:48:49,855 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:49,855 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:49,865 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:49,911 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45209888/hregion-45209888.1732506529838 2024-11-25T03:48:49,912 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:49,912 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f2a782a6615ed3a672396d4df553c9f7, NAME => 'testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741915_1093 (size=61) 2024-11-25T03:48:49,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741915_1093 (size=61) 2024-11-25T03:48:49,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741915_1093 (size=61) 2024-11-25T03:48:49,932 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:49,936 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:49,939 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2a782a6615ed3a672396d4df553c9f7 columnFamilyName a 2024-11-25T03:48:49,939 DEBUG [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:49,939 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(327): Store=f2a782a6615ed3a672396d4df553c9f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:49,940 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:49,940 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:49,941 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:49,941 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:49,941 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:49,943 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:49,945 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:49,945 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2a782a6615ed3a672396d4df553c9f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75048645, jitterRate=0.11831195652484894}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2a782a6615ed3a672396d4df553c9f7: Writing region info on filesystem at 1732506529933Initializing all the Stores at 1732506529936 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506529936Cleaning up temporary data from old regions at 1732506529941 (+5 ms)Region opened successfully at 1732506529946 (+5 ms) 2024-11-25T03:48:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f2a782a6615ed3a672396d4df553c9f7, disabling compactions & flushes 2024-11-25T03:48:49,946 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. after waiting 0 ms 2024-11-25T03:48:49,946 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:49,947 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:49,947 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f2a782a6615ed3a672396d4df553c9f7: Waiting for close lock at 1732506529946Disabling compacts and flushes for region at 1732506529946Disabling writes for close at 1732506529946Writing region close event to WAL at 1732506529947 (+1 ms)Closed at 1732506529947 2024-11-25T03:48:49,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741914_1092 (size=175) 2024-11-25T03:48:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741914_1092 (size=175) 2024-11-25T03:48:49,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741914_1092 (size=175) 2024-11-25T03:48:49,961 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:49,961 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45209888:(num 1732506529838) 2024-11-25T03:48:49,961 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:49,963 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:49,980 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964, exclude list is [], retry=0 2024-11-25T03:48:49,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:49,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:49,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:50,019 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964 2024-11-25T03:48:50,020 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:50,020 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f2a782a6615ed3a672396d4df553c9f7, NAME => 'testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:50,020 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:50,020 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,020 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,022 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,022 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2a782a6615ed3a672396d4df553c9f7 columnFamilyName a 2024-11-25T03:48:50,022 DEBUG [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:50,023 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(327): Store=f2a782a6615ed3a672396d4df553c9f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:50,023 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,024 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,025 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,025 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,031 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,032 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2a782a6615ed3a672396d4df553c9f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62395179, jitterRate=-0.07023938000202179}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:50,032 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2a782a6615ed3a672396d4df553c9f7: Writing region info on filesystem at 1732506530020Initializing all the Stores at 1732506530021 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506530021Cleaning up temporary data from old regions at 1732506530025 (+4 ms)Region opened successfully at 1732506530032 (+7 ms) 2024-11-25T03:48:50,051 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f2a782a6615ed3a672396d4df553c9f7, disabling compactions & flushes 2024-11-25T03:48:50,051 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:50,051 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:50,051 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. after waiting 0 ms 2024-11-25T03:48:50,051 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:50,052 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:50,052 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. 2024-11-25T03:48:50,052 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f2a782a6615ed3a672396d4df553c9f7: Waiting for close lock at 1732506530051Disabling compacts and flushes for region at 1732506530051Disabling writes for close at 1732506530051Writing region close event to WAL at 1732506530052 (+1 ms)Closed at 1732506530052 2024-11-25T03:48:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741916_1094 (size=1795) 2024-11-25T03:48:50,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741916_1094 (size=1795) 2024-11-25T03:48:50,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741916_1094 (size=1795) 2024-11-25T03:48:50,081 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964, size=1.8 K (1795bytes) 2024-11-25T03:48:50,082 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964 2024-11-25T03:48:50,082 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964 after 0ms 2024-11-25T03:48:50,154 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:50,154 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964 took 73ms 2024-11-25T03:48:50,156 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964 so closing down 2024-11-25T03:48:50,156 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:50,157 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732506529964.temp 2024-11-25T03:48:50,182 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000003-wal.1732506529964.temp 2024-11-25T03:48:50,183 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:50,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741917_1095 (size=1795) 2024-11-25T03:48:50,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741917_1095 (size=1795) 2024-11-25T03:48:50,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741917_1095 (size=1795) 2024-11-25T03:48:50,197 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000003-wal.1732506529964.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:50,200 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000003-wal.1732506529964.temp to hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000012 2024-11-25T03:48:50,201 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-25T03:48:50,201 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964, journal: Splitting hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964, size=1.8 K (1795bytes) at 1732506530081Finishing writing output for hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964 so closing down at 1732506530156 (+75 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000003-wal.1732506529964.temp at 1732506530182 (+26 ms)3 split writer threads finished at 1732506530183 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000003-wal.1732506529964.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732506530197 (+14 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000003-wal.1732506529964.temp to hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000012 at 1732506530201 (+4 ms)Processed 10 edits across 1 Regions in 47 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732506530201 2024-11-25T03:48:50,202 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506529964 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506529964 2024-11-25T03:48:50,203 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000012 2024-11-25T03:48:50,242 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:50,701 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:50,703 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:50,717 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506530704, exclude list is [], retry=0 2024-11-25T03:48:50,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:50,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:50,721 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:50,750 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506530704 2024-11-25T03:48:50,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:50,751 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f2a782a6615ed3a672396d4df553c9f7, NAME => 'testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:50,751 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:50,751 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,751 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,754 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,756 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2a782a6615ed3a672396d4df553c9f7 columnFamilyName a 2024-11-25T03:48:50,756 DEBUG [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:50,759 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(327): Store=f2a782a6615ed3a672396d4df553c9f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:50,760 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,761 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,763 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,764 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000012 2024-11-25T03:48:50,803 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:50,804 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000012 2024-11-25T03:48:50,804 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f2a782a6615ed3a672396d4df553c9f7 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-25T03:48:50,829 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/.tmp/a/9dbe24709c3e4a07b483cdcc28839888 is 79, key is testDatalossWhenInputError/a:x0/1732506530032/Put/seqid=0 2024-11-25T03:48:50,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741919_1097 (size=5808) 2024-11-25T03:48:50,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741919_1097 (size=5808) 2024-11-25T03:48:50,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741919_1097 (size=5808) 2024-11-25T03:48:50,840 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/.tmp/a/9dbe24709c3e4a07b483cdcc28839888 2024-11-25T03:48:50,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/.tmp/a/9dbe24709c3e4a07b483cdcc28839888 as hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/a/9dbe24709c3e4a07b483cdcc28839888 2024-11-25T03:48:50,874 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/a/9dbe24709c3e4a07b483cdcc28839888, entries=10, sequenceid=12, filesize=5.7 K 2024-11-25T03:48:50,874 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f2a782a6615ed3a672396d4df553c9f7 in 70ms, sequenceid=12, compaction requested=false; wal=null 2024-11-25T03:48:50,875 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/0000000000000000012 2024-11-25T03:48:50,876 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,876 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,880 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,885 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-25T03:48:50,887 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2a782a6615ed3a672396d4df553c9f7; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70333987, jitterRate=0.048058077692985535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:50,887 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2a782a6615ed3a672396d4df553c9f7: Writing region info on filesystem at 1732506530751Initializing all the Stores at 1732506530754 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506530754Obtaining lock to block concurrent updates at 1732506530804 (+50 ms)Preparing flush snapshotting stores in f2a782a6615ed3a672396d4df553c9f7 at 1732506530804Finished memstore snapshotting testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732506530804Flushing stores of testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7. at 1732506530804Flushing f2a782a6615ed3a672396d4df553c9f7/a: creating writer at 1732506530804Flushing f2a782a6615ed3a672396d4df553c9f7/a: appending metadata at 1732506530828 (+24 ms)Flushing f2a782a6615ed3a672396d4df553c9f7/a: closing flushed file at 1732506530828Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63dac1c6: reopening flushed file at 1732506530858 (+30 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for f2a782a6615ed3a672396d4df553c9f7 in 70ms, sequenceid=12, compaction requested=false; wal=null at 1732506530874 (+16 ms)Cleaning up temporary data from old regions at 1732506530876 (+2 ms)Region opened successfully at 1732506530887 (+11 ms) 2024-11-25T03:48:50,892 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f2a782a6615ed3a672396d4df553c9f7, NAME => 'testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:50,892 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732506529834.f2a782a6615ed3a672396d4df553c9f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:50,892 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,892 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,896 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,897 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2a782a6615ed3a672396d4df553c9f7 columnFamilyName a 2024-11-25T03:48:50,898 DEBUG [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:50,911 DEBUG [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/a/9dbe24709c3e4a07b483cdcc28839888 2024-11-25T03:48:50,911 INFO [StoreOpener-f2a782a6615ed3a672396d4df553c9f7-1 {}] regionserver.HStore(327): Store=f2a782a6615ed3a672396d4df553c9f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:50,911 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,913 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,914 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,914 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,919 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2a782a6615ed3a672396d4df553c9f7 2024-11-25T03:48:50,925 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testDatalossWhenInputError/f2a782a6615ed3a672396d4df553c9f7/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-25T03:48:50,926 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2a782a6615ed3a672396d4df553c9f7; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61348063, jitterRate=-0.08584262430667877}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T03:48:50,926 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2a782a6615ed3a672396d4df553c9f7: Writing region info on filesystem at 1732506530893Initializing all the Stores at 1732506530895 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506530895Cleaning up temporary data from old regions at 1732506530914 (+19 ms)Region opened successfully at 1732506530926 (+12 ms) 2024-11-25T03:48:50,951 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=430 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:54018 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:54822 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:39106 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:53946 [Waiting for operation #22] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:39170 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1013946533-172.17.0.2-1732506491574:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:54886 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=408 (was 408), ProcessCount=11 (was 11), AvailableMemoryMB=8558 (was 8571) 2024-11-25T03:48:50,952 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-25T03:48:50,969 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=408, ProcessCount=11, AvailableMemoryMB=8555 2024-11-25T03:48:50,969 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-25T03:48:50,989 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:51,001 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:51,002 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:51,013 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79580928, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-79580928, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:51,030 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79580928/hregion-79580928.1732506531014, exclude list is [], retry=0 2024-11-25T03:48:51,032 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:51,033 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:51,033 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:51,058 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79580928/hregion-79580928.1732506531014 2024-11-25T03:48:51,059 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:51,059 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 02a58ba50d65b6c05c704df3c6161217, NAME => 'testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:51,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741921_1099 (size=63) 2024-11-25T03:48:51,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741921_1099 (size=63) 2024-11-25T03:48:51,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741921_1099 (size=63) 2024-11-25T03:48:51,067 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:51,068 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,070 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName a 2024-11-25T03:48:51,070 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:51,070 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:51,070 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,072 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName b 2024-11-25T03:48:51,072 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:51,072 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:51,072 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,074 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName c 2024-11-25T03:48:51,074 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:51,074 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:51,074 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,075 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,075 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,076 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,076 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,077 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:51,078 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,080 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:51,080 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02a58ba50d65b6c05c704df3c6161217; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68956037, jitterRate=0.027525022625923157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:51,081 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02a58ba50d65b6c05c704df3c6161217: Writing region info on filesystem at 1732506531067Initializing all the Stores at 1732506531068 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506531068Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506531068Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506531068Cleaning up temporary data from old regions at 1732506531076 (+8 ms)Region opened successfully at 1732506531080 (+4 ms) 2024-11-25T03:48:51,081 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 02a58ba50d65b6c05c704df3c6161217, disabling compactions & flushes 2024-11-25T03:48:51,081 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:51,081 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:51,081 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. after waiting 0 ms 2024-11-25T03:48:51,081 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:51,081 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:51,081 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 02a58ba50d65b6c05c704df3c6161217: Waiting for close lock at 1732506531081Disabling compacts and flushes for region at 1732506531081Disabling writes for close at 1732506531081Writing region close event to WAL at 1732506531081Closed at 1732506531081 2024-11-25T03:48:51,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741920_1098 (size=175) 2024-11-25T03:48:51,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741920_1098 (size=175) 2024-11-25T03:48:51,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741920_1098 (size=175) 2024-11-25T03:48:51,085 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:51,085 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79580928:(num 1732506531014) 2024-11-25T03:48:51,086 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:51,087 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:51,100 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087, exclude list is [], retry=0 2024-11-25T03:48:51,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:51,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:51,103 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:51,127 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 2024-11-25T03:48:51,127 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:51,128 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 02a58ba50d65b6c05c704df3c6161217, NAME => 'testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:51,128 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:51,128 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,128 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,129 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,130 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName a 2024-11-25T03:48:51,130 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:51,131 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:51,131 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,131 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName b 2024-11-25T03:48:51,131 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:51,132 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:51,132 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,133 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName c 2024-11-25T03:48:51,133 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:51,133 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:51,133 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,134 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,135 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,135 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,135 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,136 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:51,137 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:51,138 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02a58ba50d65b6c05c704df3c6161217; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68367801, jitterRate=0.018759623169898987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:51,139 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02a58ba50d65b6c05c704df3c6161217: Writing region info on filesystem at 1732506531128Initializing all the Stores at 1732506531129 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506531129Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506531129Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506531129Cleaning up temporary data from old regions at 1732506531135 (+6 ms)Region opened successfully at 1732506531139 (+4 ms) 2024-11-25T03:48:51,143 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732506531142/Put/seqid=0 2024-11-25T03:48:51,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741923_1101 (size=4875) 2024-11-25T03:48:51,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741923_1101 (size=4875) 2024-11-25T03:48:51,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741923_1101 (size=4875) 2024-11-25T03:48:51,153 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732506531152/Put/seqid=0 2024-11-25T03:48:51,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741924_1102 (size=4875) 2024-11-25T03:48:51,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741924_1102 (size=4875) 2024-11-25T03:48:51,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741924_1102 (size=4875) 2024-11-25T03:48:51,163 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732506531163/Put/seqid=0 2024-11-25T03:48:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741925_1103 (size=4875) 2024-11-25T03:48:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741925_1103 (size=4875) 2024-11-25T03:48:51,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741925_1103 (size=4875) 2024-11-25T03:48:51,171 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 02a58ba50d65b6c05c704df3c6161217/a 2024-11-25T03:48:51,174 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-25T03:48:51,174 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T03:48:51,174 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 02a58ba50d65b6c05c704df3c6161217/a 2024-11-25T03:48:51,178 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-25T03:48:51,178 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T03:48:51,179 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 02a58ba50d65b6c05c704df3c6161217/a 2024-11-25T03:48:51,181 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-25T03:48:51,181 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-25T03:48:51,182 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 02a58ba50d65b6c05c704df3c6161217 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-25T03:48:51,196 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/.tmp/a/6358f3822e6345a8b144c40ee4eb655e is 55, key is testCompactedBulkLoadedFiles/a:a/1732506531139/Put/seqid=0 2024-11-25T03:48:51,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741926_1104 (size=5107) 2024-11-25T03:48:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741926_1104 (size=5107) 2024-11-25T03:48:51,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741926_1104 (size=5107) 2024-11-25T03:48:51,202 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/.tmp/a/6358f3822e6345a8b144c40ee4eb655e 2024-11-25T03:48:51,208 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/.tmp/a/6358f3822e6345a8b144c40ee4eb655e as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e 2024-11-25T03:48:51,214 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e, entries=1, sequenceid=4, filesize=5.0 K 2024-11-25T03:48:51,215 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 02a58ba50d65b6c05c704df3c6161217 in 33ms, sequenceid=4, compaction requested=false 2024-11-25T03:48:51,215 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 02a58ba50d65b6c05c704df3c6161217: 2024-11-25T03:48:51,217 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_ 2024-11-25T03:48:51,218 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_ 2024-11-25T03:48:51,219 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_ 2024-11-25T03:48:51,219 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 into 02a58ba50d65b6c05c704df3c6161217/a as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_ - updating store file list. 2024-11-25T03:48:51,223 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:51,224 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_ into 02a58ba50d65b6c05c704df3c6161217/a 2024-11-25T03:48:51,224 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile0 into 02a58ba50d65b6c05c704df3c6161217/a (new location: hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_) 2024-11-25T03:48:51,225 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 into 02a58ba50d65b6c05c704df3c6161217/a as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_ - updating store file list. 2024-11-25T03:48:51,229 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:51,229 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_ into 02a58ba50d65b6c05c704df3c6161217/a 2024-11-25T03:48:51,229 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile1 into 02a58ba50d65b6c05c704df3c6161217/a (new location: hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_) 2024-11-25T03:48:51,230 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 into 02a58ba50d65b6c05c704df3c6161217/a as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_ - updating store file list. 2024-11-25T03:48:51,234 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:51,234 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_ into 02a58ba50d65b6c05c704df3c6161217/a 2024-11-25T03:48:51,234 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:46839/hbase/testCompactedBulkLoadedFiles/hfile2 into 02a58ba50d65b6c05c704df3c6161217/a (new location: hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_) 2024-11-25T03:48:51,242 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-25T03:48:51,242 DEBUG [Time-limited test {}] regionserver.HStore(1541): 02a58ba50d65b6c05c704df3c6161217/a is initiating major compaction (all files) 2024-11-25T03:48:51,242 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 02a58ba50d65b6c05c704df3c6161217/a in testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:51,242 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_] into tmpdir=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/.tmp, totalSize=19.3 K 2024-11-25T03:48:51,243 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 6358f3822e6345a8b144c40ee4eb655e, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732506531139 2024-11-25T03:48:51,243 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T03:48:51,244 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T03:48:51,244 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-25T03:48:51,258 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/.tmp/a/532c267963ef49ccb390ab4d55ff7434 is 55, key is testCompactedBulkLoadedFiles/a:a/1732506531139/Put/seqid=0 2024-11-25T03:48:51,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741927_1105 (size=6154) 2024-11-25T03:48:51,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741927_1105 (size=6154) 2024-11-25T03:48:51,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741927_1105 (size=6154) 2024-11-25T03:48:51,275 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/.tmp/a/532c267963ef49ccb390ab4d55ff7434 as hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/532c267963ef49ccb390ab4d55ff7434 2024-11-25T03:48:51,282 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 02a58ba50d65b6c05c704df3c6161217/a of 02a58ba50d65b6c05c704df3c6161217 into 532c267963ef49ccb390ab4d55ff7434(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-25T03:48:51,282 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 02a58ba50d65b6c05c704df3c6161217: 2024-11-25T03:48:51,282 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T03:48:51,282 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-25T03:48:51,312 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087, size=0 (0bytes) 2024-11-25T03:48:51,312 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 might be still open, length is 0 2024-11-25T03:48:51,312 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 2024-11-25T03:48:51,312 WARN [IPC Server handler 4 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-25T03:48:51,313 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 after 1ms 2024-11-25T03:48:51,999 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:54046 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54046 dst: /127.0.0.1:34347 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34347 remote=/127.0.0.1:54046]. Total timeout mills is 60000, 59283 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:51,999 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:39220 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:33729:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39220 dst: /127.0.0.1:33729 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:51,999 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:54932 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43937:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54932 dst: /127.0.0.1:43937 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:52,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741922_1106 (size=1794) 2024-11-25T03:48:52,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741922_1106 (size=1794) 2024-11-25T03:48:54,827 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T03:48:55,315 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 after 4002ms 2024-11-25T03:48:55,391 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:55,391 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 took 4080ms 2024-11-25T03:48:55,393 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087; continuing. 2024-11-25T03:48:55,393 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 so closing down 2024-11-25T03:48:55,393 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:55,395 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732506531087.temp 2024-11-25T03:48:55,419 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000003-wal.1732506531087.temp 2024-11-25T03:48:55,420 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741928_1107 (size=800) 2024-11-25T03:48:55,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741928_1107 (size=800) 2024-11-25T03:48:55,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741928_1107 (size=800) 2024-11-25T03:48:55,426 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000003-wal.1732506531087.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:55,428 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000003-wal.1732506531087.temp to hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000008 2024-11-25T03:48:55,428 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T03:48:55,428 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087, journal: Splitting hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087, size=0 (0bytes) at 1732506531312Finishing writing output for hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 so closing down at 1732506535393 (+4081 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000003-wal.1732506531087.temp at 1732506535419 (+26 ms)3 split writer threads finished at 1732506535420 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000003-wal.1732506531087.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732506535426 (+6 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000003-wal.1732506531087.temp to hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000008 at 1732506535428 (+2 ms)Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087, size=0, length=0, corrupted=false, cancelled=false at 1732506535428 2024-11-25T03:48:55,429 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506531087 2024-11-25T03:48:55,430 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000008 2024-11-25T03:48:55,430 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:55,432 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:55,448 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506535433, exclude list is [], retry=0 2024-11-25T03:48:55,451 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:55,451 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:55,451 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:55,479 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506535433 2024-11-25T03:48:55,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:48:55,480 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 02a58ba50d65b6c05c704df3c6161217, NAME => 'testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:55,480 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:55,480 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,480 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,481 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,482 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName a 2024-11-25T03:48:55,482 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,488 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/532c267963ef49ccb390ab4d55ff7434 2024-11-25T03:48:55,493 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e 2024-11-25T03:48:55,498 DEBUG [StoreFileOpener-02a58ba50d65b6c05c704df3c6161217-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:55,498 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_ 2024-11-25T03:48:55,502 DEBUG [StoreFileOpener-02a58ba50d65b6c05c704df3c6161217-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:55,502 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_ 2024-11-25T03:48:55,506 DEBUG [StoreFileOpener-02a58ba50d65b6c05c704df3c6161217-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-25T03:48:55,507 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_ 2024-11-25T03:48:55,507 WARN [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b9dbafb 2024-11-25T03:48:55,507 WARN [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b9dbafb 2024-11-25T03:48:55,507 WARN [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b9dbafb 2024-11-25T03:48:55,507 WARN [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@2b9dbafb 2024-11-25T03:48:55,507 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_] to archive 2024-11-25T03:48:55,508 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-25T03:48:55,510 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/6358f3822e6345a8b144c40ee4eb655e 2024-11-25T03:48:55,514 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_ to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_ 2024-11-25T03:48:55,515 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_ to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_ 2024-11-25T03:48:55,516 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_ to hdfs://localhost:46839/hbase/archive/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/a/d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_ 2024-11-25T03:48:55,516 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,516 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,517 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName b 2024-11-25T03:48:55,517 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,517 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,518 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,518 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 02a58ba50d65b6c05c704df3c6161217 columnFamilyName c 2024-11-25T03:48:55,518 DEBUG [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,519 INFO [StoreOpener-02a58ba50d65b6c05c704df3c6161217-1 {}] regionserver.HStore(327): Store=02a58ba50d65b6c05c704df3c6161217/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,519 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,519 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,520 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,521 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000008 2024-11-25T03:48:55,547 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:55,549 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 02a58ba50d65b6c05c704df3c6161217 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "02a58ba50d65b6c05c704df3c6161217" family_name: "a" compaction_input: "6358f3822e6345a8b144c40ee4eb655e" compaction_input: "ba649a24cee345e6afd3ff3b5ab24065_SeqId_4_" compaction_input: "d57ae648b1f14b3194bb777b4575a0cf_SeqId_4_" compaction_input: "bfb8e44292fb4e06bf30dd2c3f6a3696_SeqId_4_" compaction_output: "532c267963ef49ccb390ab4d55ff7434" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-25T03:48:55,549 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-25T03:48:55,549 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000008 2024-11-25T03:48:55,550 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/0000000000000000008 2024-11-25T03:48:55,551 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,551 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,552 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:55,553 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 02a58ba50d65b6c05c704df3c6161217 2024-11-25T03:48:55,556 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testCompactedBulkLoadedFiles/02a58ba50d65b6c05c704df3c6161217/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-25T03:48:55,557 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 02a58ba50d65b6c05c704df3c6161217; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59821001, jitterRate=-0.10859762132167816}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:55,557 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 02a58ba50d65b6c05c704df3c6161217: Writing region info on filesystem at 1732506535480Initializing all the Stores at 1732506535481 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535481Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535481Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535481Cleaning up temporary data from old regions at 1732506535551 (+70 ms)Region opened successfully at 1732506535557 (+6 ms) 2024-11-25T03:48:55,560 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 02a58ba50d65b6c05c704df3c6161217, disabling compactions & flushes 2024-11-25T03:48:55,560 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:55,560 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:55,560 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. after waiting 0 ms 2024-11-25T03:48:55,560 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:55,561 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732506530990.02a58ba50d65b6c05c704df3c6161217. 2024-11-25T03:48:55,562 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 02a58ba50d65b6c05c704df3c6161217: Waiting for close lock at 1732506535560Disabling compacts and flushes for region at 1732506535560Disabling writes for close at 1732506535560Writing region close event to WAL at 1732506535561 (+1 ms)Closed at 1732506535561 2024-11-25T03:48:55,565 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506535433 not finished, retry = 0 2024-11-25T03:48:55,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741929_1108 (size=175) 2024-11-25T03:48:55,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741929_1108 (size=175) 2024-11-25T03:48:55,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741929_1108 (size=175) 2024-11-25T03:48:55,668 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:55,668 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732506535433) 2024-11-25T03:48:55,680 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=438 (was 430) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:46839 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_751039287_22 at /127.0.0.1:51926 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_751039287_22 at /127.0.0.1:43106 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 408), ProcessCount=11 (was 11), AvailableMemoryMB=8558 (was 8555) - AvailableMemoryMB LEAK? - 2024-11-25T03:48:55,681 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-25T03:48:55,692 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=438, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=8557 2024-11-25T03:48:55,692 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-25T03:48:55,705 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:55,707 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T03:48:55,707 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T03:48:55,710 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-77919869, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/hregion-77919869, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:55,722 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-77919869/hregion-77919869.1732506535710, exclude list is [], retry=0 2024-11-25T03:48:55,724 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:55,725 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:55,725 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:55,750 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-77919869/hregion-77919869.1732506535710 2024-11-25T03:48:55,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:55,750 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d098d91ce3ef78377fbefe72a0525667, NAME => 'testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46839/hbase 2024-11-25T03:48:55,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741931_1110 (size=67) 2024-11-25T03:48:55,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741931_1110 (size=67) 2024-11-25T03:48:55,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741931_1110 (size=67) 2024-11-25T03:48:55,758 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:55,759 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,760 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName a 2024-11-25T03:48:55,760 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,761 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,761 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,762 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName b 2024-11-25T03:48:55,762 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,762 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,763 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,764 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName c 2024-11-25T03:48:55,764 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,764 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,764 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,765 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,765 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,767 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,767 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,767 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:55,768 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,770 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T03:48:55,771 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d098d91ce3ef78377fbefe72a0525667; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71329223, jitterRate=0.0628882497549057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:55,771 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d098d91ce3ef78377fbefe72a0525667: Writing region info on filesystem at 1732506535758Initializing all the Stores at 1732506535759 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535759Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535759Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535759Cleaning up temporary data from old regions at 1732506535767 (+8 ms)Region opened successfully at 1732506535771 (+4 ms) 2024-11-25T03:48:55,771 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d098d91ce3ef78377fbefe72a0525667, disabling compactions & flushes 2024-11-25T03:48:55,771 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,771 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,771 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. after waiting 0 ms 2024-11-25T03:48:55,771 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,771 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,771 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d098d91ce3ef78377fbefe72a0525667: Waiting for close lock at 1732506535771Disabling compacts and flushes for region at 1732506535771Disabling writes for close at 1732506535771Writing region close event to WAL at 1732506535771Closed at 1732506535771 2024-11-25T03:48:55,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741930_1109 (size=175) 2024-11-25T03:48:55,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741930_1109 (size=175) 2024-11-25T03:48:55,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741930_1109 (size=175) 2024-11-25T03:48:55,775 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:48:55,775 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-77919869:(num 1732506535710) 2024-11-25T03:48:55,775 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:55,777 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:55,789 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777, exclude list is [], retry=0 2024-11-25T03:48:55,791 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:55,792 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:55,792 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:55,816 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777 2024-11-25T03:48:55,816 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831)] 2024-11-25T03:48:55,816 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d098d91ce3ef78377fbefe72a0525667, NAME => 'testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:55,816 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:55,816 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,817 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,818 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,819 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName a 2024-11-25T03:48:55,819 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,819 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,819 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,820 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName b 2024-11-25T03:48:55,820 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,820 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,821 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,821 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName c 2024-11-25T03:48:55,821 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:55,822 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:55,822 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,822 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,823 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,824 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,824 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,825 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:55,826 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:55,826 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d098d91ce3ef78377fbefe72a0525667; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61633155, jitterRate=-0.08159442245960236}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:55,826 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d098d91ce3ef78377fbefe72a0525667: Writing region info on filesystem at 1732506535817Initializing all the Stores at 1732506535818 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535818Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535818Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506535818Cleaning up temporary data from old regions at 1732506535824 (+6 ms)Region opened successfully at 1732506535826 (+2 ms) 2024-11-25T03:48:55,833 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d098d91ce3ef78377fbefe72a0525667 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-25T03:48:55,846 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/a/b4190e0bfc1b43a7b01b3ddec4ff53e7 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732506535827/Put/seqid=0 2024-11-25T03:48:55,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741933_1112 (size=5958) 2024-11-25T03:48:55,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741933_1112 (size=5958) 2024-11-25T03:48:55,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741933_1112 (size=5958) 2024-11-25T03:48:55,852 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/a/b4190e0bfc1b43a7b01b3ddec4ff53e7 2024-11-25T03:48:55,857 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/a/b4190e0bfc1b43a7b01b3ddec4ff53e7 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/a/b4190e0bfc1b43a7b01b3ddec4ff53e7 2024-11-25T03:48:55,861 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/a/b4190e0bfc1b43a7b01b3ddec4ff53e7, entries=10, sequenceid=13, filesize=5.8 K 2024-11-25T03:48:55,862 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for d098d91ce3ef78377fbefe72a0525667 in 29ms, sequenceid=13, compaction requested=false 2024-11-25T03:48:55,862 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d098d91ce3ef78377fbefe72a0525667: 2024-11-25T03:48:55,878 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d098d91ce3ef78377fbefe72a0525667, disabling compactions & flushes 2024-11-25T03:48:55,878 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,878 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,878 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. after waiting 0 ms 2024-11-25T03:48:55,878 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,879 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,879 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:48:55,879 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d098d91ce3ef78377fbefe72a0525667: Waiting for close lock at 1732506535878Disabling compacts and flushes for region at 1732506535878Disabling writes for close at 1732506535878Writing region close event to WAL at 1732506535879 (+1 ms)Closed at 1732506535879 2024-11-25T03:48:55,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741932_1111 (size=6243) 2024-11-25T03:48:55,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741932_1111 (size=6243) 2024-11-25T03:48:55,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741932_1111 (size=6243) 2024-11-25T03:48:55,899 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777, size=6.1 K (6243bytes) 2024-11-25T03:48:55,899 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777 2024-11-25T03:48:55,900 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777 after 1ms 2024-11-25T03:48:55,925 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:55,925 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777 took 26ms 2024-11-25T03:48:55,928 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777 so closing down 2024-11-25T03:48:55,928 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:48:55,929 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732506535777.temp 2024-11-25T03:48:55,954 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000003-wal.1732506535777.temp 2024-11-25T03:48:55,955 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:48:55,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741934_1113 (size=5575) 2024-11-25T03:48:55,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741934_1113 (size=5575) 2024-11-25T03:48:55,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741934_1113 (size=5575) 2024-11-25T03:48:55,960 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000003-wal.1732506535777.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T03:48:55,961 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000003-wal.1732506535777.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000035 2024-11-25T03:48:55,961 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-25T03:48:55,961 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777, size=6.1 K (6243bytes) at 1732506535899Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777 so closing down at 1732506535928 (+29 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000003-wal.1732506535777.temp at 1732506535954 (+26 ms)3 split writer threads finished at 1732506535955 (+1 ms)Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000003-wal.1732506535777.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732506535960 (+5 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000003-wal.1732506535777.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000035 at 1732506535961 (+1 ms)Processed 32 edits across 1 Regions in 35 ms; skipped=2; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732506535961 2024-11-25T03:48:55,963 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535777 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506535777 2024-11-25T03:48:55,964 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000035 2024-11-25T03:48:55,964 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:48:55,965 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:48:55,980 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966, exclude list is [], retry=0 2024-11-25T03:48:55,982 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:48:55,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:48:55,983 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:48:56,010 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 2024-11-25T03:48:56,010 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39943:39943),(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539)] 2024-11-25T03:48:56,010 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d098d91ce3ef78377fbefe72a0525667, NAME => 'testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667.', STARTKEY => '', ENDKEY => ''} 2024-11-25T03:48:56,010 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:48:56,010 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,010 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,012 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,013 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName a 2024-11-25T03:48:56,013 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:56,018 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/a/b4190e0bfc1b43a7b01b3ddec4ff53e7 2024-11-25T03:48:56,019 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:56,019 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,020 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName b 2024-11-25T03:48:56,020 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:56,020 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:56,020 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,021 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName c 2024-11-25T03:48:56,021 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:48:56,021 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:48:56,021 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,023 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,024 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000035 2024-11-25T03:48:56,047 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:48:56,049 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000035 2024-11-25T03:48:56,049 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d098d91ce3ef78377fbefe72a0525667 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-25T03:48:56,062 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/b/2cac523a5a554aff8d47a67016b0b89f is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732506535862/Put/seqid=0 2024-11-25T03:48:56,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741936_1115 (size=5958) 2024-11-25T03:48:56,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741936_1115 (size=5958) 2024-11-25T03:48:56,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741936_1115 (size=5958) 2024-11-25T03:48:56,069 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/b/2cac523a5a554aff8d47a67016b0b89f 2024-11-25T03:48:56,088 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/c/3d58e464441047b59a207435616c8975 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732506535869/Put/seqid=0 2024-11-25T03:48:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741937_1116 (size=5958) 2024-11-25T03:48:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741937_1116 (size=5958) 2024-11-25T03:48:56,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741937_1116 (size=5958) 2024-11-25T03:48:56,094 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/c/3d58e464441047b59a207435616c8975 2024-11-25T03:48:56,099 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/b/2cac523a5a554aff8d47a67016b0b89f as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/b/2cac523a5a554aff8d47a67016b0b89f 2024-11-25T03:48:56,104 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/b/2cac523a5a554aff8d47a67016b0b89f, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T03:48:56,105 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/c/3d58e464441047b59a207435616c8975 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/c/3d58e464441047b59a207435616c8975 2024-11-25T03:48:56,110 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/c/3d58e464441047b59a207435616c8975, entries=10, sequenceid=35, filesize=5.8 K 2024-11-25T03:48:56,110 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d098d91ce3ef78377fbefe72a0525667 in 61ms, sequenceid=35, compaction requested=false; wal=null 2024-11-25T03:48:56,111 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000035 2024-11-25T03:48:56,112 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,112 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,112 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:48:56,114 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:48:56,116 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-25T03:48:56,117 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d098d91ce3ef78377fbefe72a0525667; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59586971, jitterRate=-0.11208494007587433}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:48:56,117 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d098d91ce3ef78377fbefe72a0525667: Writing region info on filesystem at 1732506536010Initializing all the Stores at 1732506536011 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506536011Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506536012 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506536012Obtaining lock to block concurrent updates at 1732506536049 (+37 ms)Preparing flush snapshotting stores in d098d91ce3ef78377fbefe72a0525667 at 1732506536049Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732506536049Flushing stores of testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. at 1732506536049Flushing d098d91ce3ef78377fbefe72a0525667/b: creating writer at 1732506536049Flushing d098d91ce3ef78377fbefe72a0525667/b: appending metadata at 1732506536062 (+13 ms)Flushing d098d91ce3ef78377fbefe72a0525667/b: closing flushed file at 1732506536062Flushing d098d91ce3ef78377fbefe72a0525667/c: creating writer at 1732506536073 (+11 ms)Flushing d098d91ce3ef78377fbefe72a0525667/c: appending metadata at 1732506536087 (+14 ms)Flushing d098d91ce3ef78377fbefe72a0525667/c: closing flushed file at 1732506536087Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16bcd2f7: reopening flushed file at 1732506536098 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37750c1c: reopening flushed file at 1732506536104 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for d098d91ce3ef78377fbefe72a0525667 in 61ms, sequenceid=35, compaction requested=false; wal=null at 1732506536110 (+6 ms)Cleaning up temporary data from old regions at 1732506536112 (+2 ms)Region opened successfully at 1732506536117 (+5 ms) 2024-11-25T03:48:56,174 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966, size=0 (0bytes) 2024-11-25T03:48:56,174 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 might be still open, length is 0 2024-11-25T03:48:56,174 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 2024-11-25T03:48:56,175 WARN [IPC Server handler 2 on default port 46839 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-25T03:48:56,175 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 after 1ms 2024-11-25T03:48:57,292 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-25T03:48:57,292 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:57,294 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-25T03:48:57,295 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-25T03:48:57,997 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:43182 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43937:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43182 dst: /127.0.0.1:43937 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43937 remote=/127.0.0.1:43182]. Total timeout mills is 60000, 58142 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:57,998 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:52002 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:33729:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52002 dst: /127.0.0.1:33729 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:57,998 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1967718664_22 at /127.0.0.1:55894 [Receiving block BP-1013946533-172.17.0.2-1732506491574:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34347:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55894 dst: /127.0.0.1:34347 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:48:58,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741935_1117 (size=5567) 2024-11-25T03:48:58,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741935_1117 (size=5567) 2024-11-25T03:48:58,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741935_1117 (size=5567) 2024-11-25T03:49:00,176 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 after 4002ms 2024-11-25T03:49:00,212 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:49:00,213 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 took 4039ms 2024-11-25T03:49:00,215 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966; continuing. 2024-11-25T03:49:00,216 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 so closing down 2024-11-25T03:49:00,216 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-25T03:49:00,217 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732506535966.temp 2024-11-25T03:49:00,250 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000037-wal.1732506535966.temp 2024-11-25T03:49:00,250 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-25T03:49:00,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741938_1118 (size=5575) 2024-11-25T03:49:00,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741938_1118 (size=5575) 2024-11-25T03:49:00,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741938_1118 (size=5575) 2024-11-25T03:49:00,261 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000037-wal.1732506535966.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-25T03:49:00,262 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000037-wal.1732506535966.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000066 2024-11-25T03:49:00,262 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966, size=0, length=0, corrupted=false, cancelled=false 2024-11-25T03:49:00,263 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966, journal: Splitting hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966, size=0 (0bytes) at 1732506536174Finishing writing output for hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 so closing down at 1732506540216 (+4042 ms)Creating recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000037-wal.1732506535966.temp at 1732506540250 (+34 ms)3 split writer threads finished at 1732506540250Closed recovered edits writer path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000037-wal.1732506535966.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732506540261 (+11 ms)Rename recovered edits hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000037-wal.1732506535966.temp to hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000066 at 1732506540262 (+1 ms)Processed 30 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966, size=0, length=0, corrupted=false, cancelled=false at 1732506540262 2024-11-25T03:49:00,264 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 to hdfs://localhost:46839/hbase/oldWALs/wal.1732506535966 2024-11-25T03:49:00,265 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000066 2024-11-25T03:49:00,265 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-25T03:49:00,267 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:46839/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705, archiveDir=hdfs://localhost:46839/hbase/oldWALs, maxLogs=32 2024-11-25T03:49:00,280 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506540267, exclude list is [], retry=0 2024-11-25T03:49:00,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33729,DS-f4401b48-ff05-43fe-8ad5-7ff5e3fb7bae,DISK] 2024-11-25T03:49:00,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34347,DS-46eb1ee3-53d8-481e-a269-4af1c9a64e74,DISK] 2024-11-25T03:49:00,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43937,DS-5f8a72f1-bbd5-439f-b0ba-cc2535449969,DISK] 2024-11-25T03:49:00,335 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506540267 2024-11-25T03:49:00,335 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32831:32831),(127.0.0.1/127.0.0.1:40539:40539),(127.0.0.1/127.0.0.1:39943:39943)] 2024-11-25T03:49:00,336 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T03:49:00,337 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,338 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName a 2024-11-25T03:49:00,338 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:49:00,346 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/a/b4190e0bfc1b43a7b01b3ddec4ff53e7 2024-11-25T03:49:00,346 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:49:00,346 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,347 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName b 2024-11-25T03:49:00,347 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:49:00,382 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/b/2cac523a5a554aff8d47a67016b0b89f 2024-11-25T03:49:00,382 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:49:00,382 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,383 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d098d91ce3ef78377fbefe72a0525667 columnFamilyName c 2024-11-25T03:49:00,383 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T03:49:00,390 DEBUG [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/c/3d58e464441047b59a207435616c8975 2024-11-25T03:49:00,390 INFO [StoreOpener-d098d91ce3ef78377fbefe72a0525667-1 {}] regionserver.HStore(327): Store=d098d91ce3ef78377fbefe72a0525667/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T03:49:00,391 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,392 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,395 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000066 2024-11-25T03:49:00,427 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-25T03:49:00,431 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000066 2024-11-25T03:49:00,432 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d098d91ce3ef78377fbefe72a0525667 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-25T03:49:00,439 INFO [master/d73269e94d72:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-25T03:49:00,439 INFO [master/d73269e94d72:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-25T03:49:00,455 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/a/85a80cf353bf4fc1bc97532f2bec607b is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732506536122/Put/seqid=0 2024-11-25T03:49:00,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741940_1120 (size=5958) 2024-11-25T03:49:00,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741940_1120 (size=5958) 2024-11-25T03:49:00,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741940_1120 (size=5958) 2024-11-25T03:49:00,465 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/a/85a80cf353bf4fc1bc97532f2bec607b 2024-11-25T03:49:00,491 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/b/00ce62332cf84f9b9fd038636e0cc32d is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732506536129/Put/seqid=0 2024-11-25T03:49:00,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741941_1121 (size=5958) 2024-11-25T03:49:00,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741941_1121 (size=5958) 2024-11-25T03:49:00,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741941_1121 (size=5958) 2024-11-25T03:49:00,498 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/b/00ce62332cf84f9b9fd038636e0cc32d 2024-11-25T03:49:00,518 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/c/516ab1527ca94d548937b05ee36b9eb2 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732506536134/Put/seqid=0 2024-11-25T03:49:00,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741942_1122 (size=5958) 2024-11-25T03:49:00,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741942_1122 (size=5958) 2024-11-25T03:49:00,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741942_1122 (size=5958) 2024-11-25T03:49:00,528 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/c/516ab1527ca94d548937b05ee36b9eb2 2024-11-25T03:49:00,534 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/a/85a80cf353bf4fc1bc97532f2bec607b as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/a/85a80cf353bf4fc1bc97532f2bec607b 2024-11-25T03:49:00,542 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/a/85a80cf353bf4fc1bc97532f2bec607b, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T03:49:00,543 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/b/00ce62332cf84f9b9fd038636e0cc32d as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/b/00ce62332cf84f9b9fd038636e0cc32d 2024-11-25T03:49:00,548 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/b/00ce62332cf84f9b9fd038636e0cc32d, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T03:49:00,550 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/.tmp/c/516ab1527ca94d548937b05ee36b9eb2 as hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/c/516ab1527ca94d548937b05ee36b9eb2 2024-11-25T03:49:00,555 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/c/516ab1527ca94d548937b05ee36b9eb2, entries=10, sequenceid=66, filesize=5.8 K 2024-11-25T03:49:00,556 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d098d91ce3ef78377fbefe72a0525667 in 125ms, sequenceid=66, compaction requested=false; wal=null 2024-11-25T03:49:00,556 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/0000000000000000066 2024-11-25T03:49:00,558 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,558 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,558 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-25T03:49:00,560 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d098d91ce3ef78377fbefe72a0525667 2024-11-25T03:49:00,562 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/hbase/data/default/testReplayEditsWrittenViaHRegion/d098d91ce3ef78377fbefe72a0525667/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-25T03:49:00,563 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d098d91ce3ef78377fbefe72a0525667; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63745175, jitterRate=-0.05012287199497223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-25T03:49:00,564 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d098d91ce3ef78377fbefe72a0525667: Writing region info on filesystem at 1732506540336Initializing all the Stores at 1732506540336Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506540336Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506540337 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732506540337Obtaining lock to block concurrent updates at 1732506540432 (+95 ms)Preparing flush snapshotting stores in d098d91ce3ef78377fbefe72a0525667 at 1732506540432Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732506540432Flushing stores of testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. at 1732506540432Flushing d098d91ce3ef78377fbefe72a0525667/a: creating writer at 1732506540432Flushing d098d91ce3ef78377fbefe72a0525667/a: appending metadata at 1732506540454 (+22 ms)Flushing d098d91ce3ef78377fbefe72a0525667/a: closing flushed file at 1732506540454Flushing d098d91ce3ef78377fbefe72a0525667/b: creating writer at 1732506540471 (+17 ms)Flushing d098d91ce3ef78377fbefe72a0525667/b: appending metadata at 1732506540490 (+19 ms)Flushing d098d91ce3ef78377fbefe72a0525667/b: closing flushed file at 1732506540490Flushing d098d91ce3ef78377fbefe72a0525667/c: creating writer at 1732506540503 (+13 ms)Flushing d098d91ce3ef78377fbefe72a0525667/c: appending metadata at 1732506540517 (+14 ms)Flushing d098d91ce3ef78377fbefe72a0525667/c: closing flushed file at 1732506540517Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@737a833a: reopening flushed file at 1732506540533 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dc87249: reopening flushed file at 1732506540542 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b369628: reopening flushed file at 1732506540549 (+7 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for d098d91ce3ef78377fbefe72a0525667 in 125ms, sequenceid=66, compaction requested=false; wal=null at 1732506540556 (+7 ms)Cleaning up temporary data from old regions at 1732506540558 (+2 ms)Region opened successfully at 1732506540564 (+6 ms) 2024-11-25T03:49:00,579 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d098d91ce3ef78377fbefe72a0525667, disabling compactions & flushes 2024-11-25T03:49:00,579 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:49:00,579 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:49:00,579 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. after waiting 0 ms 2024-11-25T03:49:00,579 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:49:00,582 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732506535706.d098d91ce3ef78377fbefe72a0525667. 2024-11-25T03:49:00,582 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d098d91ce3ef78377fbefe72a0525667: Waiting for close lock at 1732506540579Disabling compacts and flushes for region at 1732506540579Disabling writes for close at 1732506540579Writing region close event to WAL at 1732506540582 (+3 ms)Closed at 1732506540582 2024-11-25T03:49:00,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741939_1119 (size=175) 2024-11-25T03:49:00,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741939_1119 (size=175) 2024-11-25T03:49:00,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741939_1119 (size=175) 2024-11-25T03:49:00,588 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-25T03:49:00,588 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732506540267) 2024-11-25T03:49:00,603 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=441 (was 438) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-388053780_22 at /127.0.0.1:55938 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1775838448) connection to localhost/127.0.0.1:46839 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-388053780_22 at /127.0.0.1:43218 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-388053780_22 at /127.0.0.1:52038 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:46839 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=8559 (was 8557) - AvailableMemoryMB LEAK? - 2024-11-25T03:49:00,604 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-25T03:49:00,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T03:49:00,604 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:49:00,604 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:49:00,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:49:00,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:49:00,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T03:49:00,605 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T03:49:00,605 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=639412996, stopped=false 2024-11-25T03:49:00,606 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=d73269e94d72,41803,1732506496940 2024-11-25T03:49:00,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:49:00,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:49:00,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:49:00,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:49:00,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T03:49:00,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:49:00,627 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:49:00,627 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T03:49:00,628 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:49:00,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:49:00,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:49:00,628 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,33171,1732506497711' ***** 2024-11-25T03:49:00,628 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:49:00,628 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:49:00,628 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:49:00,628 INFO [RS:0;d73269e94d72:33171 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:49:00,628 INFO [RS:0;d73269e94d72:33171 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:49:00,628 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:49:00,629 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,33171,1732506497711 2024-11-25T03:49:00,629 INFO [RS:0;d73269e94d72:33171 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:49:00,629 INFO [RS:0;d73269e94d72:33171 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;d73269e94d72:33171. 2024-11-25T03:49:00,629 DEBUG [RS:0;d73269e94d72:33171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:49:00,629 DEBUG [RS:0;d73269e94d72:33171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:49:00,629 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T03:49:00,629 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1325): Online Regions={1b485cca92b597d29ec36b9eadec9462=testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462.} 2024-11-25T03:49:00,629 DEBUG [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1351): Waiting on 1b485cca92b597d29ec36b9eadec9462 2024-11-25T03:49:00,629 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:49:00,629 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b485cca92b597d29ec36b9eadec9462, disabling compactions & flushes 2024-11-25T03:49:00,629 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:49:00,629 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T03:49:00,629 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:49:00,629 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. after waiting 0 ms 2024-11-25T03:49:00,629 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:49:00,630 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'd73269e94d72,38451,1732506497897' ***** 2024-11-25T03:49:00,630 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T03:49:00,630 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T03:49:00,630 INFO [RS:2;d73269e94d72:38451 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T03:49:00,630 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T03:49:00,630 INFO [RS:2;d73269e94d72:38451 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T03:49:00,631 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(959): stopping server d73269e94d72,38451,1732506497897 2024-11-25T03:49:00,631 INFO [RS:2;d73269e94d72:38451 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:49:00,631 INFO [RS:2;d73269e94d72:38451 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;d73269e94d72:38451. 2024-11-25T03:49:00,631 DEBUG [RS:2;d73269e94d72:38451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T03:49:00,631 DEBUG [RS:2;d73269e94d72:38451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:49:00,631 INFO [RS:2;d73269e94d72:38451 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:49:00,631 INFO [RS:2;d73269e94d72:38451 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:49:00,631 INFO [RS:2;d73269e94d72:38451 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:49:00,631 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T03:49:00,636 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T03:49:00,636 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T03:49:00,636 DEBUG [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T03:49:00,636 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T03:49:00,636 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T03:49:00,636 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T03:49:00,636 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T03:49:00,637 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T03:49:00,637 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-25T03:49:00,639 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/default/testReplayEditsAfterRegionMovedWithMultiCF/1b485cca92b597d29ec36b9eadec9462/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-25T03:49:00,640 INFO [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:49:00,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b485cca92b597d29ec36b9eadec9462: Waiting for close lock at 1732506540629Running coprocessor pre-close hooks at 1732506540629Disabling compacts and flushes for region at 1732506540629Disabling writes for close at 1732506540629Writing region close event to WAL at 1732506540631 (+2 ms)Running coprocessor post-close hooks at 1732506540640 (+9 ms)Closed at 1732506540640 2024-11-25T03:49:00,640 DEBUG [RS_CLOSE_REGION-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462. 2024-11-25T03:49:00,644 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/info/0f6ee139b1a248dbb5aa03d5949fee63 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732506513684.1b485cca92b597d29ec36b9eadec9462./info:regioninfo/1732506518471/Put/seqid=0 2024-11-25T03:49:00,651 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:49:00,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741943_1123 (size=6778) 2024-11-25T03:49:00,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741943_1123 (size=6778) 2024-11-25T03:49:00,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741943_1123 (size=6778) 2024-11-25T03:49:00,660 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/info/0f6ee139b1a248dbb5aa03d5949fee63 2024-11-25T03:49:00,666 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/.tmp/info/0f6ee139b1a248dbb5aa03d5949fee63 as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/info/0f6ee139b1a248dbb5aa03d5949fee63 2024-11-25T03:49:00,671 INFO [regionserver/d73269e94d72:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:49:00,672 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/info/0f6ee139b1a248dbb5aa03d5949fee63, entries=8, sequenceid=23, filesize=6.6 K 2024-11-25T03:49:00,673 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 36ms, sequenceid=23, compaction requested=false 2024-11-25T03:49:00,698 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-25T03:49:00,698 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T03:49:00,698 INFO [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T03:49:00,699 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732506540636Running coprocessor pre-close hooks at 1732506540636Disabling compacts and flushes for region at 1732506540636Disabling writes for close at 1732506540636Obtaining lock to block concurrent updates at 1732506540637 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732506540637Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1732506540637Flushing stores of hbase:meta,,1.1588230740 at 1732506540638 (+1 ms)Flushing 1588230740/info: creating writer at 1732506540638Flushing 1588230740/info: appending metadata at 1732506540643 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732506540644 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d9ef588: reopening flushed file at 1732506540665 (+21 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 36ms, sequenceid=23, compaction requested=false at 1732506540673 (+8 ms)Writing region close event to WAL at 1732506540692 (+19 ms)Running coprocessor post-close hooks at 1732506540698 (+6 ms)Closed at 1732506540698 2024-11-25T03:49:00,699 DEBUG [RS_CLOSE_META-regionserver/d73269e94d72:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T03:49:00,829 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,33171,1732506497711; all regions closed. 2024-11-25T03:49:00,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741834_1010 (size=3621) 2024-11-25T03:49:00,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741834_1010 (size=3621) 2024-11-25T03:49:00,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741834_1010 (size=3621) 2024-11-25T03:49:00,836 DEBUG [RS:0;d73269e94d72:33171 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C33171%2C1732506497711:(num 1732506499467) 2024-11-25T03:49:00,836 DEBUG [RS:0;d73269e94d72:33171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T03:49:00,836 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(976): stopping server d73269e94d72,38451,1732506497897; all regions closed. 2024-11-25T03:49:00,836 INFO [RS:0;d73269e94d72:33171 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:49:00,837 INFO [RS:0;d73269e94d72:33171 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33171 2024-11-25T03:49:00,837 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:49:00,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741893_1071 (size=2625) 2024-11-25T03:49:00,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741893_1071 (size=2625) 2024-11-25T03:49:00,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741893_1071 (size=2625) 2024-11-25T03:49:00,844 DEBUG [RS:2;d73269e94d72:38451 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs 2024-11-25T03:49:00,844 INFO [RS:2;d73269e94d72:38451 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C38451%2C1732506497897.meta:.meta(num 1732506517427) 2024-11-25T03:49:00,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741833_1009 (size=175) 2024-11-25T03:49:00,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741833_1009 (size=175) 2024-11-25T03:49:00,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741833_1009 (size=175) 2024-11-25T03:49:00,850 DEBUG [RS:2;d73269e94d72:38451 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/oldWALs 2024-11-25T03:49:00,850 INFO [RS:2;d73269e94d72:38451 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL d73269e94d72%2C38451%2C1732506497897:(num 1732506499467) 2024-11-25T03:49:00,850 DEBUG [RS:2;d73269e94d72:38451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T03:49:00,850 INFO [RS:2;d73269e94d72:38451 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T03:49:00,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:49:00,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,33171,1732506497711 2024-11-25T03:49:00,850 INFO [RS:0;d73269e94d72:33171 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:49:00,850 INFO [RS:2;d73269e94d72:38451 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:49:00,850 INFO [RS:2;d73269e94d72:38451 {}] hbase.ChoreService(370): Chore service for: regionserver/d73269e94d72:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T03:49:00,850 INFO [RS:2;d73269e94d72:38451 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:49:00,851 INFO [RS:2;d73269e94d72:38451 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38451 2024-11-25T03:49:00,851 INFO [regionserver/d73269e94d72:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:49:00,852 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,33171,1732506497711] 2024-11-25T03:49:00,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/d73269e94d72,38451,1732506497897 2024-11-25T03:49:00,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T03:49:00,871 INFO [RS:2;d73269e94d72:38451 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:49:00,872 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,33171,1732506497711 already deleted, retry=false 2024-11-25T03:49:00,872 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,33171,1732506497711 expired; onlineServers=1 2024-11-25T03:49:00,882 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [d73269e94d72,38451,1732506497897] 2024-11-25T03:49:00,892 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/d73269e94d72,38451,1732506497897 already deleted, retry=false 2024-11-25T03:49:00,892 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; d73269e94d72,38451,1732506497897 expired; onlineServers=0 2024-11-25T03:49:00,892 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'd73269e94d72,41803,1732506496940' ***** 2024-11-25T03:49:00,892 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T03:49:00,893 INFO [M:0;d73269e94d72:41803 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T03:49:00,893 INFO [M:0;d73269e94d72:41803 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T03:49:00,893 DEBUG [M:0;d73269e94d72:41803 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T03:49:00,893 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T03:49:00,893 DEBUG [M:0;d73269e94d72:41803 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T03:49:00,893 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506499078 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.large.0-1732506499078,5,FailOnTimeoutGroup] 2024-11-25T03:49:00,893 INFO [M:0;d73269e94d72:41803 {}] hbase.ChoreService(370): Chore service for: master/d73269e94d72:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T03:49:00,893 INFO [M:0;d73269e94d72:41803 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T03:49:00,893 DEBUG [M:0;d73269e94d72:41803 {}] master.HMaster(1795): Stopping service threads 2024-11-25T03:49:00,893 INFO [M:0;d73269e94d72:41803 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T03:49:00,893 DEBUG [master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506499082 {}] cleaner.HFileCleaner(306): Exit Thread[master/d73269e94d72:0:becomeActiveMaster-HFileCleaner.small.0-1732506499082,5,FailOnTimeoutGroup] 2024-11-25T03:49:00,894 INFO [M:0;d73269e94d72:41803 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T03:49:00,894 INFO [M:0;d73269e94d72:41803 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T03:49:00,894 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T03:49:00,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T03:49:00,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T03:49:00,903 DEBUG [M:0;d73269e94d72:41803 {}] zookeeper.ZKUtil(347): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T03:49:00,903 WARN [M:0;d73269e94d72:41803 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T03:49:00,904 INFO [M:0;d73269e94d72:41803 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/.lastflushedseqids 2024-11-25T03:49:00,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741944_1124 (size=119) 2024-11-25T03:49:00,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741944_1124 (size=119) 2024-11-25T03:49:00,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741944_1124 (size=119) 2024-11-25T03:49:00,919 INFO [M:0;d73269e94d72:41803 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T03:49:00,919 INFO [M:0;d73269e94d72:41803 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T03:49:00,919 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T03:49:00,919 INFO [M:0;d73269e94d72:41803 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:49:00,919 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:49:00,919 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T03:49:00,919 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:49:00,919 INFO [M:0;d73269e94d72:41803 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.38 KB heapSize=102.70 KB 2024-11-25T03:49:00,939 DEBUG [M:0;d73269e94d72:41803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2177dd5d51ca47338a373dfa2e36d534 is 82, key is hbase:meta,,1/info:regioninfo/1732506517712/Put/seqid=0 2024-11-25T03:49:00,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:49:00,961 INFO [RS:0;d73269e94d72:33171 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:49:00,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33171-0x101701b280d0001, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:49:00,961 INFO [RS:0;d73269e94d72:33171 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,33171,1732506497711; zookeeper connection closed. 2024-11-25T03:49:00,972 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b535d0f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b535d0f 2024-11-25T03:49:00,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741945_1125 (size=6063) 2024-11-25T03:49:00,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741945_1125 (size=6063) 2024-11-25T03:49:00,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741945_1125 (size=6063) 2024-11-25T03:49:00,975 INFO [M:0;d73269e94d72:41803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2177dd5d51ca47338a373dfa2e36d534 2024-11-25T03:49:00,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:49:00,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38451-0x101701b280d0003, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:49:00,982 INFO [RS:2;d73269e94d72:38451 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:49:00,982 INFO [RS:2;d73269e94d72:38451 {}] regionserver.HRegionServer(1031): Exiting; stopping=d73269e94d72,38451,1732506497897; zookeeper connection closed. 2024-11-25T03:49:00,984 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@782ad100 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@782ad100 2024-11-25T03:49:00,985 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T03:49:01,001 DEBUG [M:0;d73269e94d72:41803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/afcb0f1ecda6449bb7d5e62583fea37e is 1075, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732506514197/Put/seqid=0 2024-11-25T03:49:01,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741946_1126 (size=7906) 2024-11-25T03:49:01,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741946_1126 (size=7906) 2024-11-25T03:49:01,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741946_1126 (size=7906) 2024-11-25T03:49:01,026 INFO [M:0;d73269e94d72:41803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.16 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/afcb0f1ecda6449bb7d5e62583fea37e 2024-11-25T03:49:01,033 INFO [M:0;d73269e94d72:41803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for afcb0f1ecda6449bb7d5e62583fea37e 2024-11-25T03:49:01,054 DEBUG [M:0;d73269e94d72:41803 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e2264823752a4467bfd6bf90e3a26422 is 69, key is d73269e94d72,33171,1732506497711/rs:state/1732506499170/Put/seqid=0 2024-11-25T03:49:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741947_1127 (size=5445) 2024-11-25T03:49:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741947_1127 (size=5445) 2024-11-25T03:49:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741947_1127 (size=5445) 2024-11-25T03:49:01,065 INFO [M:0;d73269e94d72:41803 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e2264823752a4467bfd6bf90e3a26422 2024-11-25T03:49:01,071 INFO [M:0;d73269e94d72:41803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2264823752a4467bfd6bf90e3a26422 2024-11-25T03:49:01,071 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2177dd5d51ca47338a373dfa2e36d534 as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2177dd5d51ca47338a373dfa2e36d534 2024-11-25T03:49:01,076 INFO [M:0;d73269e94d72:41803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2177dd5d51ca47338a373dfa2e36d534, entries=14, sequenceid=207, filesize=5.9 K 2024-11-25T03:49:01,078 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/afcb0f1ecda6449bb7d5e62583fea37e as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/afcb0f1ecda6449bb7d5e62583fea37e 2024-11-25T03:49:01,083 INFO [M:0;d73269e94d72:41803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for afcb0f1ecda6449bb7d5e62583fea37e 2024-11-25T03:49:01,083 INFO [M:0;d73269e94d72:41803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/afcb0f1ecda6449bb7d5e62583fea37e, entries=21, sequenceid=207, filesize=7.7 K 2024-11-25T03:49:01,084 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e2264823752a4467bfd6bf90e3a26422 as hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e2264823752a4467bfd6bf90e3a26422 2024-11-25T03:49:01,090 INFO [M:0;d73269e94d72:41803 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for e2264823752a4467bfd6bf90e3a26422 2024-11-25T03:49:01,090 INFO [M:0;d73269e94d72:41803 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46839/user/jenkins/test-data/e4f5a15d-229c-4a36-0ad3-bca1421698d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e2264823752a4467bfd6bf90e3a26422, entries=3, sequenceid=207, filesize=5.3 K 2024-11-25T03:49:01,091 INFO [M:0;d73269e94d72:41803 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 172ms, sequenceid=207, compaction requested=false 2024-11-25T03:49:01,092 INFO [M:0;d73269e94d72:41803 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T03:49:01,092 DEBUG [M:0;d73269e94d72:41803 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732506540919Disabling compacts and flushes for region at 1732506540919Disabling writes for close at 1732506540919Obtaining lock to block concurrent updates at 1732506540919Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732506540919Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85386, getHeapSize=105096, getOffHeapSize=0, getCellsCount=248 at 1732506540919Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732506540920 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732506540920Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732506540938 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732506540938Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732506540981 (+43 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732506541001 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732506541001Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732506541034 (+33 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732506541053 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732506541053Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50226c77: reopening flushed file at 1732506541071 (+18 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69570f86: reopening flushed file at 1732506541077 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eb612f2: reopening flushed file at 1732506541083 (+6 ms)Finished flush of dataSize ~83.38 KB/85386, heapSize ~102.40 KB/104856, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 172ms, sequenceid=207, compaction requested=false at 1732506541091 (+8 ms)Writing region close event to WAL at 1732506541092 (+1 ms)Closed at 1732506541092 2024-11-25T03:49:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33729 is added to blk_1073741830_1006 (size=102150) 2024-11-25T03:49:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34347 is added to blk_1073741830_1006 (size=102150) 2024-11-25T03:49:01,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43937 is added to blk_1073741830_1006 (size=102150) 2024-11-25T03:49:01,098 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T03:49:01,098 INFO [M:0;d73269e94d72:41803 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T03:49:01,099 INFO [M:0;d73269e94d72:41803 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41803 2024-11-25T03:49:01,100 INFO [M:0;d73269e94d72:41803 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T03:49:01,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:49:01,211 INFO [M:0;d73269e94d72:41803 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T03:49:01,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41803-0x101701b280d0000, quorum=127.0.0.1:57852, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T03:49:01,217 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506529443 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506529443 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732506519989/wal.1732506529443 (inode 16688) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-25T03:49:01,220 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519593 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519593 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732506518939/wal.1732506519593 (inode 16665) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-25T03:49:01,222 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732506509081/wal.1732506509193 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:49:01,224 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506530704 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506530704 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732506529833/wal.1732506530704 (inode 16714) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-25T03:49:01,225 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732506530988/wal.1732506531087 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:49:01,225 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732506535705/wal.1732506535966 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:49:01,225 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732506501834/wal.1732506501928 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T03:49:01,228 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal.1732506501654 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal.1732506501654 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732506501272/wal.1732506501654 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-25T03:49:01,231 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal.1732506500990 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal.1732506500990 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732506500535/wal.1732506500990 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1967718664_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-25T03:49:01,246 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6f0cddd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:49:01,249 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@25cdce3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:49:01,249 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:49:01,249 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1846121a{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:49:01,250 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7161498b{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,STOPPED} 2024-11-25T03:49:01,253 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T03:49:01,253 WARN [BP-1013946533-172.17.0.2-1732506491574 heartbeating to localhost/127.0.0.1:46839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T03:49:01,253 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T03:49:01,253 WARN [BP-1013946533-172.17.0.2-1732506491574 heartbeating to localhost/127.0.0.1:46839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1013946533-172.17.0.2-1732506491574 (Datanode Uuid 4c6f3230-8958-4157-8900-e7142a6de5d9) service to localhost/127.0.0.1:46839 2024-11-25T03:49:01,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data5/current/BP-1013946533-172.17.0.2-1732506491574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:49:01,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data6/current/BP-1013946533-172.17.0.2-1732506491574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:49:01,257 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T03:49:01,261 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@44df5d9d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:49:01,261 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@56fbb88e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:49:01,262 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:49:01,262 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40a3a086{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:49:01,262 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b034666{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,STOPPED} 2024-11-25T03:49:01,264 WARN [BP-1013946533-172.17.0.2-1732506491574 heartbeating to localhost/127.0.0.1:46839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T03:49:01,264 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T03:49:01,264 WARN [BP-1013946533-172.17.0.2-1732506491574 heartbeating to localhost/127.0.0.1:46839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1013946533-172.17.0.2-1732506491574 (Datanode Uuid 4315fb89-1f23-4d4f-8c92-1866ea4ff287) service to localhost/127.0.0.1:46839 2024-11-25T03:49:01,264 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T03:49:01,265 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data3/current/BP-1013946533-172.17.0.2-1732506491574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:49:01,265 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data4/current/BP-1013946533-172.17.0.2-1732506491574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:49:01,265 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T03:49:01,270 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@45b57af8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-25T03:49:01,270 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@461b5ea8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:49:01,270 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:49:01,270 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e9da5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:49:01,271 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5cf706e7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,STOPPED} 2024-11-25T03:49:01,277 WARN [BP-1013946533-172.17.0.2-1732506491574 heartbeating to localhost/127.0.0.1:46839 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-25T03:49:01,277 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-25T03:49:01,278 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-25T03:49:01,278 WARN [BP-1013946533-172.17.0.2-1732506491574 heartbeating to localhost/127.0.0.1:46839 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1013946533-172.17.0.2-1732506491574 (Datanode Uuid d70f36d4-ff73-46a6-b535-9c9b5b1f19f5) service to localhost/127.0.0.1:46839 2024-11-25T03:49:01,278 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data1/current/BP-1013946533-172.17.0.2-1732506491574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:49:01,279 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/cluster_6682261b-d2a5-71af-29e4-2c455fc4595a/data/data2/current/BP-1013946533-172.17.0.2-1732506491574 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-25T03:49:01,279 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-25T03:49:01,286 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5608f426{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-25T03:49:01,287 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@63efd3a6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-25T03:49:01,287 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-25T03:49:01,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3192a19b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-25T03:49:01,287 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e7ae814{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9501b4a5-df88-bd90-d67f-2fc023e8b7c0/hadoop.log.dir/,STOPPED} 2024-11-25T03:49:01,298 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T03:49:01,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down